Compress setting sql columns into output specification
authoreileen <emcnaughton@wikimedia.org>
Mon, 3 Dec 2018 03:51:34 +0000 (16:51 +1300)
committereileen <emcnaughton@wikimedia.org>
Thu, 13 Dec 2018 05:17:48 +0000 (18:17 +1300)
CRM/Export/BAO/Export.php
CRM/Export/BAO/ExportProcessor.php
tests/phpunit/CRM/Export/BAO/ExportTest.php

index c6c651af8ede00512978dc94898bd24677bba137..dd6ca1295f0f03d6d295c17f94e7a04fda54e97e 100644 (file)
@@ -287,7 +287,6 @@ class CRM_Export_BAO_Export {
         if (!array_key_exists($column, $returnProperties)) {
           $returnProperties[$column] = 1;
           $column = $column == 'id' ? 'civicrm_primary_id' : $column;
-          $processor->setColumnAsCalculationOnly($column);
           $exportParams['merge_same_address']['temp_columns'][$column] = 1;
         }
       }
@@ -320,7 +319,6 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
         if (!array_key_exists($column, $returnProperties)) {
           $returnProperties[$column] = 1;
           $exportParams['postal_mailing_export']['temp_columns'][$column] = 1;
-          $processor->setColumnAsCalculationOnly($column);
         }
       }
     }
@@ -444,11 +442,8 @@ INSERT INTO {$componentTable} SELECT distinct gc.contact_id FROM civicrm_group_c
     list($outputColumns, $metadata) = self::getExportStructureArrays($returnProperties, $processor);
     $headerRows = $processor->getHeaderRows();
 
-
     // add payment headers if required
     if ($addPaymentHeader && $processor->isExportPaymentFields()) {
-      // @todo rather than do this for every single row do it before the loop starts.
-      // where other header definitions take place.
       $headerRows = array_merge($headerRows, $processor->getPaymentHeaders());
       foreach (array_keys($processor->getPaymentHeaders()) as $paymentHdr) {
         $processor->setSqlColumnDefn($paymentHdr);
@@ -1228,46 +1223,26 @@ WHERE  {$whereClause}";
    *       yet find a way to comment them for posterity.
    */
   public static function getExportStructureArrays($returnProperties, $processor) {
-    $metadata = $outputColumns = array();
-    $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
-    $imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
+    $outputColumns = $metadata = array();
     $queryFields = $processor->getQueryFields();
     foreach ($returnProperties as $key => $value) {
       if (($key != 'location' || !is_array($value)) && !$processor->isRelationshipTypeKey($key)) {
         $outputColumns[$key] = $value;
         $processor->addOutputSpecification($key);
-        $processor->setSqlColumnDefn($key);
       }
       elseif ($processor->isRelationshipTypeKey($key)) {
         $outputColumns[$key] = $value;
-        $field = $key;
         foreach ($value as $relationField => $relationValue) {
           // below block is same as primary block (duplicate)
           if (isset($queryFields[$relationField]['title'])) {
-            if (!$processor->isHouseholdMergeRelationshipTypeKey($field)) {
-              // Do not add to header row if we are only generating for merge reasons.
-              $processor->addOutputSpecification($relationField, $key);
-            }
-            $processor->setSqlColumnDefn($field . '-' . $relationField);
+            $processor->addOutputSpecification($relationField, $key);
           }
           elseif (is_array($relationValue) && $relationField == 'location') {
             // fix header for location type case
             foreach ($relationValue as $ltype => $val) {
               foreach (array_keys($val) as $fld) {
                 $type = explode('-', $fld);
-
-                $hdr = "{$ltype}-" . $queryFields[$type[0]]['title'];
-
-                if (!empty($type[1])) {
-                  if (CRM_Utils_Array::value(0, $type) == 'phone') {
-                    $hdr .= "-" . CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $type[1]);
-                  }
-                  elseif (CRM_Utils_Array::value(0, $type) == 'im') {
-                    $hdr .= "-" . CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $type[1]);
-                  }
-                }
-                $processor->addOutputSpecification($field, $key, $ltype, CRM_Utils_Array::value(1, $type));
-                $processor->setSqlColumnDefn($field . '-' . $hdr);
+                $processor->addOutputSpecification($type[0], $key, $ltype, CRM_Utils_Array::value(1, $type));
               }
             }
           }
@@ -1279,31 +1254,13 @@ WHERE  {$whereClause}";
             $type = explode('-', $locationFieldName);
 
             $actualDBFieldName = $type[0];
-            $outputFieldName = $locationType . '-' . $queryFields[$actualDBFieldName]['title'];
             $daoFieldName = CRM_Utils_String::munge($locationType) . '-' . $actualDBFieldName;
 
             if (!empty($type[1])) {
               $daoFieldName .= "-" . $type[1];
-              if ($actualDBFieldName == 'phone') {
-                $outputFieldName .= "-" . CRM_Utils_Array::value($type[1], $phoneTypes);
-              }
-              elseif ($actualDBFieldName == 'im') {
-                $outputFieldName .= "-" . CRM_Utils_Array::value($type[1], $imProviders);
-              }
-            }
-            if ($type[0] == 'im_provider') {
-              // Warning: shame inducing hack.
-              $metadata[$daoFieldName]['pseudoconstant']['var'] = 'imProviders';
-            }
-
-            $processor->addOutputSpecification($outputFieldName, NULL, $locationType, CRM_Utils_Array::value(1, $type));
-            $processor->setSqlColumnDefn($outputFieldName);
-            if ($actualDBFieldName == 'country' || $actualDBFieldName == 'world_region') {
-              $metadata[$daoFieldName] = array('context' => 'country');
-            }
-            if ($actualDBFieldName == 'state_province') {
-              $metadata[$daoFieldName] = array('context' => 'province');
             }
+            $processor->addOutputSpecification($actualDBFieldName, NULL, $locationType, CRM_Utils_Array::value(1, $type));
+            $metadata[$daoFieldName] = $processor->getMetaDataForField($actualDBFieldName);
             $outputColumns[$daoFieldName] = TRUE;
           }
         }
index 711b7e51c4f6b98e6ad8cf5718a57fa4a5369538..553b741d1fb3b1b63e14d2bf68e9a2ee7b4d7508 100644 (file)
@@ -292,6 +292,12 @@ class CRM_Export_BAO_ExportProcessor {
    * @param array $queryFields
    */
   public function setQueryFields($queryFields) {
+    // legacy hacks - we add these to queryFields because this
+    // pseudometadata is currently required.
+    $queryFields['im_provider']['pseudoconstant']['var'] = 'imProviders';
+    $queryFields['country']['context'] = 'country';
+    $queryFields['world_region']['context'] = 'country';
+    $queryFields['state_province']['context'] = 'province';
     $this->queryFields = $queryFields;
   }
 
@@ -450,32 +456,54 @@ class CRM_Export_BAO_ExportProcessor {
    * @param int $entityTypeID phone_type_id or provider_id for phone or im fields.
    */
   public function addOutputSpecification($key, $relationshipType = NULL, $locationType = NULL, $entityTypeID = NULL) {
-    $label = $this->getHeaderForRow($key);
-    $labelPrefix = $fieldPrefix = [];
-    if ($relationshipType) {
-      $labelPrefix[] = $this->getRelationshipTypes()[$relationshipType];
-      $fieldPrefix[] = $relationshipType;
-    }
-    if ($locationType) {
-      $labelPrefix[] = $fieldPrefix[] = $locationType;
-    }
+    $entityLabel = '';
     if ($entityTypeID) {
       if ($key === 'phone') {
-        $labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $entityTypeID);
+        $entityLabel = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $entityTypeID);
       }
       if ($key === 'im') {
-        $labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $entityTypeID);
+        $entityLabel = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $entityTypeID);
       }
     }
-    $index = $this->getMungedFieldName(($fieldPrefix ? (implode('-', $fieldPrefix)  . '-') : '') . $key);
-    $this->outputSpecification[$index]['header'] = ($labelPrefix ? (implode('-', $labelPrefix) . '-') : '') . $label;
+
+    // These oddly constructed keys are for legacy reasons. Altering them will affect test success
+    // but in time it may be good to rationalise them.
+    $label = $this->getOutputSpecificationLabel($key, $relationshipType, $locationType, $entityLabel);
+    $index = $this->getOutputSpecificationIndex($key, $relationshipType, $locationType, $entityLabel);
+    $fieldKey = $this->getOutputSpecificationFieldKey($key, $relationshipType, $locationType, $entityLabel);
+
+    $this->outputSpecification[$index]['header'] = $label;
+    $this->outputSpecification[$index]['sql_columns'] = $this->getSqlColumnDefinition($fieldKey, $key);
+
+    if ($relationshipType && $this->isHouseholdMergeRelationshipTypeKey($relationshipType)) {
+      $this->setColumnAsCalculationOnly($index);
+    }
+    $this->outputSpecification[$index]['metadata'] = $this->getMetaDataForField($key);
+  }
+
+  /**
+   * Get the metadata for the given field.
+   *
+   * @param $key
+   *
+   * @return array
+   */
+  public function getMetaDataForField($key) {
+    $mappings = ['contact_id' => 'id'];
+    if (isset($this->getQueryFields()[$key])) {
+      return $this->getQueryFields()[$key];
+    }
+    if (isset($mappings[$key])) {
+      return $this->getQueryFields()[$mappings[$key]];
+    }
+    return [];
   }
 
   /**
    * @param $key
    */
   public function setSqlColumnDefn($key) {
-    $this->outputSpecification[$this->getMungedFieldName($key)]['sql_columns'] = $this->getSqlColumnDefinition($key);
+    $this->outputSpecification[$this->getMungedFieldName($key)]['sql_columns'] = $this->getSqlColumnDefinition($key, $this->getMungedFieldName($key));
   }
 
   /**
@@ -515,6 +543,16 @@ class CRM_Export_BAO_ExportProcessor {
     return $sqlColumns;
   }
 
+  /**
+   * @return array
+   */
+  public function getMetadata() {
+    $metadata = [];
+    foreach ($this->outputSpecification as $key => $spec) {
+      $metadata[$key] = $spec['metadata'];
+    }
+    return $metadata;
+  }
 
   /**
    * Build the row for output.
@@ -973,17 +1011,17 @@ class CRM_Export_BAO_ExportProcessor {
   /**
    * Get the sql column definition for the given field.
    *
-   * @param $field
+   * @param string $fieldName
+   * @param string $columnName
    *
    * @return mixed
    */
-  public function getSqlColumnDefinition($field) {
-    $fieldName = $this->getMungedFieldName($field);
+  public function getSqlColumnDefinition($fieldName, $columnName) {
 
     // early exit for master_id, CRM-12100
     // in the DB it is an ID, but in the export, we retrive the display_name of the master record
     // also for current_employer, CRM-16939
-    if ($fieldName == 'master_id' || $fieldName == 'current_employer') {
+    if ($columnName == 'master_id' || $columnName == 'current_employer') {
       return "$fieldName varchar(128)";
     }
 
@@ -995,11 +1033,11 @@ class CRM_Export_BAO_ExportProcessor {
     $queryFields = $this->getQueryFields();
     $lookUp = ['prefix_id', 'suffix_id'];
     // set the sql columns
-    if (isset($queryFields[$field]['type'])) {
-      switch ($queryFields[$field]['type']) {
+    if (isset($queryFields[$columnName]['type'])) {
+      switch ($queryFields[$columnName]['type']) {
         case CRM_Utils_Type::T_INT:
         case CRM_Utils_Type::T_BOOLEAN:
-          if (in_array($field, $lookUp)) {
+          if (in_array($columnName, $lookUp)) {
             return "$fieldName varchar(255)";
           }
           else {
@@ -1007,8 +1045,8 @@ class CRM_Export_BAO_ExportProcessor {
           }
 
         case CRM_Utils_Type::T_STRING:
-          if (isset($queryFields[$field]['maxlength'])) {
-            return "$fieldName varchar({$queryFields[$field]['maxlength']})";
+          if (isset($queryFields[$columnName]['maxlength'])) {
+            return "$fieldName varchar({$queryFields[$columnName]['maxlength']})";
           }
           else {
             return "$fieldName varchar(255)";
@@ -1052,12 +1090,12 @@ class CRM_Export_BAO_ExportProcessor {
         }
         else {
           // set the sql columns for custom data
-          if (isset($queryFields[$field]['data_type'])) {
+          if (isset($queryFields[$columnName]['data_type'])) {
 
-            switch ($queryFields[$field]['data_type']) {
+            switch ($queryFields[$columnName]['data_type']) {
               case 'String':
                 // May be option labels, which could be up to 512 characters
-                $length = max(512, CRM_Utils_Array::value('text_length', $queryFields[$field]));
+                $length = max(512, CRM_Utils_Array::value('text_length', $queryFields[$columnName]));
                 return "$fieldName varchar($length)";
 
               case 'Country':
@@ -1094,4 +1132,89 @@ class CRM_Export_BAO_ExportProcessor {
     return $fieldName;
   }
 
+  /**
+   * In order to respect the history of this class we need to index kinda illogically.
+   *
+   * On the bright side - this stuff is tested within a nano-byte of it's life.
+   *
+   * e.g '2-a-b_Home-City'
+   *
+   * @param string $key
+   * @param string $relationshipType
+   * @param string $locationType
+   * @param $entityLabel
+   *
+   * @return string
+   */
+  protected function getOutputSpecificationIndex($key, $relationshipType, $locationType, $entityLabel) {
+    if ($locationType || $entityLabel || $key === 'im') {
+      // Just cos that's the history...
+      if ($key !== 'master_id') {
+        $key = $this->getHeaderForRow($key);
+      }
+    }
+    if (!$relationshipType || $key !== 'id') {
+      $key = $this->getMungedFieldName($key);
+    }
+    return $this->getMungedFieldName(
+      ($relationshipType ? ($relationshipType . '_') : '')
+      . ($locationType ? ($locationType . '_') : '')
+      . $key
+      . ($entityLabel ? ('_' . $entityLabel) : '')
+    );
+  }
+
+  /**
+   * Get the compiled label for the column.
+   *
+   * e.g 'Gender', 'Employee Of-Home-city'
+   *
+   * @param string $key
+   * @param string $relationshipType
+   * @param string $locationType
+   * @param string $entityLabel
+   *
+   * @return string
+   */
+  protected function getOutputSpecificationLabel($key, $relationshipType, $locationType, $entityLabel) {
+    return ($relationshipType ? $this->getRelationshipTypes()[$relationshipType] . '-' : '')
+      . ($locationType ? $locationType . '-' : '')
+      . $this->getHeaderForRow($key)
+      . ($entityLabel ? '-' . $entityLabel : '');
+  }
+
+  /**
+   * Get the mysql field name key.
+   *
+   * This key is locked in by tests but the reasons for the specific conventions -
+   * ie. headings are used for keying fields in some cases, are likely
+   * accidental rather than deliberate.
+   *
+   * This key is used for the output sql array.
+   *
+   * @param string $key
+   * @param $relationshipType
+   * @param $locationType
+   * @param $entityLabel
+   *
+   * @return string
+   */
+  protected function getOutputSpecificationFieldKey($key, $relationshipType, $locationType, $entityLabel) {
+    if ($relationshipType || $entityLabel || $key === 'im') {
+      if ($key !== 'state_province' && $key !== 'id') {
+        $key = $this->getHeaderForRow($key);
+      }
+    }
+    if (!$relationshipType || $key !== 'id') {
+      $key = $this->getMungedFieldName($key);
+    }
+    $fieldKey = $this->getMungedFieldName(
+      ($relationshipType ? ($relationshipType . '_') : '')
+      . ($locationType ? ($locationType . '_') : '')
+      . $key
+      . ($entityLabel ? ('_' . $entityLabel) : '')
+    );
+    return $fieldKey;
+  }
+
 }
index f1615704466c9004d41ebe81ddf0da4387a37f2b..bc3dd2743ab474d143f72602ca9c802b7cdf6e3e 100644 (file)
@@ -628,7 +628,7 @@ class CRM_Export_BAO_ExportTest extends CiviUnitTestCase {
 
     list($tableName, $sqlColumns) = $this->doExport($selectedFields, $this->contactIDs[1]);
     $this->assertEquals([
-      'billing_city' => 'billing_city text',
+      'billing_city' => 'billing_city varchar(64)',
       'custom_1' => 'custom_1 varchar(255)',
     ], $sqlColumns);
 
@@ -736,99 +736,99 @@ class CRM_Export_BAO_ExportTest extends CiviUnitTestCase {
 
     $this->assertEquals([
       'billing_im_provider' => 'billing_im_provider text',
-      'billing_im_screen_name' => 'billing_im_screen_name text',
-      'billing_im_screen_name_jabber' => 'billing_im_screen_name_jabber text',
-      'billing_im_screen_name_skype' => 'billing_im_screen_name_skype text',
-      'billing_im_screen_name_yahoo' => 'billing_im_screen_name_yahoo text',
+      'billing_im_screen_name' => 'billing_im_screen_name varchar(64)',
+      'billing_im_screen_name_jabber' => 'billing_im_screen_name_jabber varchar(64)',
+      'billing_im_screen_name_skype' => 'billing_im_screen_name_skype varchar(64)',
+      'billing_im_screen_name_yahoo' => 'billing_im_screen_name_yahoo varchar(64)',
       'home_im_provider' => 'home_im_provider text',
-      'home_im_screen_name' => 'home_im_screen_name text',
-      'home_im_screen_name_jabber' => 'home_im_screen_name_jabber text',
-      'home_im_screen_name_skype' => 'home_im_screen_name_skype text',
-      'home_im_screen_name_yahoo' => 'home_im_screen_name_yahoo text',
+      'home_im_screen_name' => 'home_im_screen_name varchar(64)',
+      'home_im_screen_name_jabber' => 'home_im_screen_name_jabber varchar(64)',
+      'home_im_screen_name_skype' => 'home_im_screen_name_skype varchar(64)',
+      'home_im_screen_name_yahoo' => 'home_im_screen_name_yahoo varchar(64)',
       'main_im_provider' => 'main_im_provider text',
-      'main_im_screen_name' => 'main_im_screen_name text',
-      'main_im_screen_name_jabber' => 'main_im_screen_name_jabber text',
-      'main_im_screen_name_skype' => 'main_im_screen_name_skype text',
-      'main_im_screen_name_yahoo' => 'main_im_screen_name_yahoo text',
+      'main_im_screen_name' => 'main_im_screen_name varchar(64)',
+      'main_im_screen_name_jabber' => 'main_im_screen_name_jabber varchar(64)',
+      'main_im_screen_name_skype' => 'main_im_screen_name_skype varchar(64)',
+      'main_im_screen_name_yahoo' => 'main_im_screen_name_yahoo varchar(64)',
       'other_im_provider' => 'other_im_provider text',
-      'other_im_screen_name' => 'other_im_screen_name text',
-      'other_im_screen_name_jabber' => 'other_im_screen_name_jabber text',
-      'other_im_screen_name_skype' => 'other_im_screen_name_skype text',
-      'other_im_screen_name_yahoo' => 'other_im_screen_name_yahoo text',
+      'other_im_screen_name' => 'other_im_screen_name varchar(64)',
+      'other_im_screen_name_jabber' => 'other_im_screen_name_jabber varchar(64)',
+      'other_im_screen_name_skype' => 'other_im_screen_name_skype varchar(64)',
+      'other_im_screen_name_yahoo' => 'other_im_screen_name_yahoo varchar(64)',
       'im_provider' => 'im_provider text',
-      'im' => 'im varchar(64)',
+      'im_screen_name' => 'im_screen_name varchar(64)',
       'contact_id' => 'contact_id varchar(255)',
       '2_a_b_im_provider' => '2_a_b_im_provider text',
-      '2_a_b_billing_im_screen_name' => '2_a_b_billing_im_screen_name text',
-      '2_a_b_billing_im_screen_name_jabber' => '2_a_b_billing_im_screen_name_jabber text',
-      '2_a_b_billing_im_screen_name_skype' => '2_a_b_billing_im_screen_name_skype text',
-      '2_a_b_billing_im_screen_name_yahoo' => '2_a_b_billing_im_screen_name_yahoo text',
-      '2_a_b_home_im_screen_name' => '2_a_b_home_im_screen_name text',
-      '2_a_b_home_im_screen_name_jabber' => '2_a_b_home_im_screen_name_jabber text',
-      '2_a_b_home_im_screen_name_skype' => '2_a_b_home_im_screen_name_skype text',
-      '2_a_b_home_im_screen_name_yahoo' => '2_a_b_home_im_screen_name_yahoo text',
-      '2_a_b_main_im_screen_name' => '2_a_b_main_im_screen_name text',
-      '2_a_b_main_im_screen_name_jabber' => '2_a_b_main_im_screen_name_jabber text',
-      '2_a_b_main_im_screen_name_skype' => '2_a_b_main_im_screen_name_skype text',
-      '2_a_b_main_im_screen_name_yahoo' => '2_a_b_main_im_screen_name_yahoo text',
-      '2_a_b_other_im_screen_name' => '2_a_b_other_im_screen_name text',
-      '2_a_b_other_im_screen_name_jabber' => '2_a_b_other_im_screen_name_jabber text',
-      '2_a_b_other_im_screen_name_skype' => '2_a_b_other_im_screen_name_skype text',
-      '2_a_b_other_im_screen_name_yahoo' => '2_a_b_other_im_screen_name_yahoo text',
-      '2_a_b_im' => '2_a_b_im text',
+      '2_a_b_billing_im_screen_name' => '2_a_b_billing_im_screen_name varchar(64)',
+      '2_a_b_billing_im_screen_name_jabber' => '2_a_b_billing_im_screen_name_jabber varchar(64)',
+      '2_a_b_billing_im_screen_name_skype' => '2_a_b_billing_im_screen_name_skype varchar(64)',
+      '2_a_b_billing_im_screen_name_yahoo' => '2_a_b_billing_im_screen_name_yahoo varchar(64)',
+      '2_a_b_home_im_screen_name' => '2_a_b_home_im_screen_name varchar(64)',
+      '2_a_b_home_im_screen_name_jabber' => '2_a_b_home_im_screen_name_jabber varchar(64)',
+      '2_a_b_home_im_screen_name_skype' => '2_a_b_home_im_screen_name_skype varchar(64)',
+      '2_a_b_home_im_screen_name_yahoo' => '2_a_b_home_im_screen_name_yahoo varchar(64)',
+      '2_a_b_main_im_screen_name' => '2_a_b_main_im_screen_name varchar(64)',
+      '2_a_b_main_im_screen_name_jabber' => '2_a_b_main_im_screen_name_jabber varchar(64)',
+      '2_a_b_main_im_screen_name_skype' => '2_a_b_main_im_screen_name_skype varchar(64)',
+      '2_a_b_main_im_screen_name_yahoo' => '2_a_b_main_im_screen_name_yahoo varchar(64)',
+      '2_a_b_other_im_screen_name' => '2_a_b_other_im_screen_name varchar(64)',
+      '2_a_b_other_im_screen_name_jabber' => '2_a_b_other_im_screen_name_jabber varchar(64)',
+      '2_a_b_other_im_screen_name_skype' => '2_a_b_other_im_screen_name_skype varchar(64)',
+      '2_a_b_other_im_screen_name_yahoo' => '2_a_b_other_im_screen_name_yahoo varchar(64)',
+      '2_a_b_im_screen_name' => '2_a_b_im_screen_name varchar(64)',
       '8_a_b_im_provider' => '8_a_b_im_provider text',
-      '8_a_b_billing_im_screen_name' => '8_a_b_billing_im_screen_name text',
-      '8_a_b_billing_im_screen_name_jabber' => '8_a_b_billing_im_screen_name_jabber text',
-      '8_a_b_billing_im_screen_name_skype' => '8_a_b_billing_im_screen_name_skype text',
-      '8_a_b_billing_im_screen_name_yahoo' => '8_a_b_billing_im_screen_name_yahoo text',
-      '8_a_b_home_im_screen_name' => '8_a_b_home_im_screen_name text',
-      '8_a_b_home_im_screen_name_jabber' => '8_a_b_home_im_screen_name_jabber text',
-      '8_a_b_home_im_screen_name_skype' => '8_a_b_home_im_screen_name_skype text',
-      '8_a_b_home_im_screen_name_yahoo' => '8_a_b_home_im_screen_name_yahoo text',
-      '8_a_b_main_im_screen_name' => '8_a_b_main_im_screen_name text',
-      '8_a_b_main_im_screen_name_jabber' => '8_a_b_main_im_screen_name_jabber text',
-      '8_a_b_main_im_screen_name_skype' => '8_a_b_main_im_screen_name_skype text',
-      '8_a_b_main_im_screen_name_yahoo' => '8_a_b_main_im_screen_name_yahoo text',
-      '8_a_b_other_im_screen_name' => '8_a_b_other_im_screen_name text',
-      '8_a_b_other_im_screen_name_jabber' => '8_a_b_other_im_screen_name_jabber text',
-      '8_a_b_other_im_screen_name_skype' => '8_a_b_other_im_screen_name_skype text',
-      '8_a_b_other_im_screen_name_yahoo' => '8_a_b_other_im_screen_name_yahoo text',
-      '8_a_b_im' => '8_a_b_im text',
+      '8_a_b_billing_im_screen_name' => '8_a_b_billing_im_screen_name varchar(64)',
+      '8_a_b_billing_im_screen_name_jabber' => '8_a_b_billing_im_screen_name_jabber varchar(64)',
+      '8_a_b_billing_im_screen_name_skype' => '8_a_b_billing_im_screen_name_skype varchar(64)',
+      '8_a_b_billing_im_screen_name_yahoo' => '8_a_b_billing_im_screen_name_yahoo varchar(64)',
+      '8_a_b_home_im_screen_name' => '8_a_b_home_im_screen_name varchar(64)',
+      '8_a_b_home_im_screen_name_jabber' => '8_a_b_home_im_screen_name_jabber varchar(64)',
+      '8_a_b_home_im_screen_name_skype' => '8_a_b_home_im_screen_name_skype varchar(64)',
+      '8_a_b_home_im_screen_name_yahoo' => '8_a_b_home_im_screen_name_yahoo varchar(64)',
+      '8_a_b_main_im_screen_name' => '8_a_b_main_im_screen_name varchar(64)',
+      '8_a_b_main_im_screen_name_jabber' => '8_a_b_main_im_screen_name_jabber varchar(64)',
+      '8_a_b_main_im_screen_name_skype' => '8_a_b_main_im_screen_name_skype varchar(64)',
+      '8_a_b_main_im_screen_name_yahoo' => '8_a_b_main_im_screen_name_yahoo varchar(64)',
+      '8_a_b_other_im_screen_name' => '8_a_b_other_im_screen_name varchar(64)',
+      '8_a_b_other_im_screen_name_jabber' => '8_a_b_other_im_screen_name_jabber varchar(64)',
+      '8_a_b_other_im_screen_name_skype' => '8_a_b_other_im_screen_name_skype varchar(64)',
+      '8_a_b_other_im_screen_name_yahoo' => '8_a_b_other_im_screen_name_yahoo varchar(64)',
+      '8_a_b_im_screen_name' => '8_a_b_im_screen_name varchar(64)',
       '5_a_b_im_provider' => '5_a_b_im_provider text',
-      '5_a_b_billing_im_screen_name' => '5_a_b_billing_im_screen_name text',
-      '5_a_b_billing_im_screen_name_jabber' => '5_a_b_billing_im_screen_name_jabber text',
-      '5_a_b_billing_im_screen_name_skype' => '5_a_b_billing_im_screen_name_skype text',
-      '5_a_b_billing_im_screen_name_yahoo' => '5_a_b_billing_im_screen_name_yahoo text',
-      '5_a_b_home_im_screen_name' => '5_a_b_home_im_screen_name text',
-      '5_a_b_home_im_screen_name_jabber' => '5_a_b_home_im_screen_name_jabber text',
-      '5_a_b_home_im_screen_name_skype' => '5_a_b_home_im_screen_name_skype text',
-      '5_a_b_home_im_screen_name_yahoo' => '5_a_b_home_im_screen_name_yahoo text',
-      '5_a_b_main_im_screen_name' => '5_a_b_main_im_screen_name text',
-      '5_a_b_main_im_screen_name_jabber' => '5_a_b_main_im_screen_name_jabber text',
-      '5_a_b_main_im_screen_name_skype' => '5_a_b_main_im_screen_name_skype text',
-      '5_a_b_main_im_screen_name_yahoo' => '5_a_b_main_im_screen_name_yahoo text',
-      '5_a_b_other_im_screen_name' => '5_a_b_other_im_screen_name text',
-      '5_a_b_other_im_screen_name_jabber' => '5_a_b_other_im_screen_name_jabber text',
-      '5_a_b_other_im_screen_name_skype' => '5_a_b_other_im_screen_name_skype text',
-      '5_a_b_other_im_screen_name_yahoo' => '5_a_b_other_im_screen_name_yahoo text',
-      '5_a_b_im' => '5_a_b_im text',
+      '5_a_b_billing_im_screen_name' => '5_a_b_billing_im_screen_name varchar(64)',
+      '5_a_b_billing_im_screen_name_jabber' => '5_a_b_billing_im_screen_name_jabber varchar(64)',
+      '5_a_b_billing_im_screen_name_skype' => '5_a_b_billing_im_screen_name_skype varchar(64)',
+      '5_a_b_billing_im_screen_name_yahoo' => '5_a_b_billing_im_screen_name_yahoo varchar(64)',
+      '5_a_b_home_im_screen_name' => '5_a_b_home_im_screen_name varchar(64)',
+      '5_a_b_home_im_screen_name_jabber' => '5_a_b_home_im_screen_name_jabber varchar(64)',
+      '5_a_b_home_im_screen_name_skype' => '5_a_b_home_im_screen_name_skype varchar(64)',
+      '5_a_b_home_im_screen_name_yahoo' => '5_a_b_home_im_screen_name_yahoo varchar(64)',
+      '5_a_b_main_im_screen_name' => '5_a_b_main_im_screen_name varchar(64)',
+      '5_a_b_main_im_screen_name_jabber' => '5_a_b_main_im_screen_name_jabber varchar(64)',
+      '5_a_b_main_im_screen_name_skype' => '5_a_b_main_im_screen_name_skype varchar(64)',
+      '5_a_b_main_im_screen_name_yahoo' => '5_a_b_main_im_screen_name_yahoo varchar(64)',
+      '5_a_b_other_im_screen_name' => '5_a_b_other_im_screen_name varchar(64)',
+      '5_a_b_other_im_screen_name_jabber' => '5_a_b_other_im_screen_name_jabber varchar(64)',
+      '5_a_b_other_im_screen_name_skype' => '5_a_b_other_im_screen_name_skype varchar(64)',
+      '5_a_b_other_im_screen_name_yahoo' => '5_a_b_other_im_screen_name_yahoo varchar(64)',
+      '5_a_b_im_screen_name' => '5_a_b_im_screen_name varchar(64)',
       'whare_kai_im_provider' => 'whare_kai_im_provider text',
-      'whare_kai_im_screen_name' => 'whare_kai_im_screen_name text',
-      'whare_kai_im_screen_name_jabber' => 'whare_kai_im_screen_name_jabber text',
-      'whare_kai_im_screen_name_skype' => 'whare_kai_im_screen_name_skype text',
-      'whare_kai_im_screen_name_yahoo' => 'whare_kai_im_screen_name_yahoo text',
-      '2_a_b_whare_kai_im_screen_name' => '2_a_b_whare_kai_im_screen_name text',
-      '2_a_b_whare_kai_im_screen_name_jabber' => '2_a_b_whare_kai_im_screen_name_jabber text',
-      '2_a_b_whare_kai_im_screen_name_skype' => '2_a_b_whare_kai_im_screen_name_skype text',
-      '2_a_b_whare_kai_im_screen_name_yahoo' => '2_a_b_whare_kai_im_screen_name_yahoo text',
-      '8_a_b_whare_kai_im_screen_name' => '8_a_b_whare_kai_im_screen_name text',
-      '8_a_b_whare_kai_im_screen_name_jabber' => '8_a_b_whare_kai_im_screen_name_jabber text',
-      '8_a_b_whare_kai_im_screen_name_skype' => '8_a_b_whare_kai_im_screen_name_skype text',
-      '8_a_b_whare_kai_im_screen_name_yahoo' => '8_a_b_whare_kai_im_screen_name_yahoo text',
-      '5_a_b_whare_kai_im_screen_name' => '5_a_b_whare_kai_im_screen_name text',
-      '5_a_b_whare_kai_im_screen_name_jabber' => '5_a_b_whare_kai_im_screen_name_jabber text',
-      '5_a_b_whare_kai_im_screen_name_skype' => '5_a_b_whare_kai_im_screen_name_skype text',
-      '5_a_b_whare_kai_im_screen_name_yahoo' => '5_a_b_whare_kai_im_screen_name_yahoo text',
+      'whare_kai_im_screen_name' => 'whare_kai_im_screen_name varchar(64)',
+      'whare_kai_im_screen_name_jabber' => 'whare_kai_im_screen_name_jabber varchar(64)',
+      'whare_kai_im_screen_name_skype' => 'whare_kai_im_screen_name_skype varchar(64)',
+      'whare_kai_im_screen_name_yahoo' => 'whare_kai_im_screen_name_yahoo varchar(64)',
+      '2_a_b_whare_kai_im_screen_name' => '2_a_b_whare_kai_im_screen_name varchar(64)',
+      '2_a_b_whare_kai_im_screen_name_jabber' => '2_a_b_whare_kai_im_screen_name_jabber varchar(64)',
+      '2_a_b_whare_kai_im_screen_name_skype' => '2_a_b_whare_kai_im_screen_name_skype varchar(64)',
+      '2_a_b_whare_kai_im_screen_name_yahoo' => '2_a_b_whare_kai_im_screen_name_yahoo varchar(64)',
+      '8_a_b_whare_kai_im_screen_name' => '8_a_b_whare_kai_im_screen_name varchar(64)',
+      '8_a_b_whare_kai_im_screen_name_jabber' => '8_a_b_whare_kai_im_screen_name_jabber varchar(64)',
+      '8_a_b_whare_kai_im_screen_name_skype' => '8_a_b_whare_kai_im_screen_name_skype varchar(64)',
+      '8_a_b_whare_kai_im_screen_name_yahoo' => '8_a_b_whare_kai_im_screen_name_yahoo varchar(64)',
+      '5_a_b_whare_kai_im_screen_name' => '5_a_b_whare_kai_im_screen_name varchar(64)',
+      '5_a_b_whare_kai_im_screen_name_jabber' => '5_a_b_whare_kai_im_screen_name_jabber varchar(64)',
+      '5_a_b_whare_kai_im_screen_name_skype' => '5_a_b_whare_kai_im_screen_name_skype varchar(64)',
+      '5_a_b_whare_kai_im_screen_name_yahoo' => '5_a_b_whare_kai_im_screen_name_yahoo varchar(64)',
     ], $sqlColumns);
 
   }
@@ -906,39 +906,39 @@ class CRM_Export_BAO_ExportTest extends CiviUnitTestCase {
 
     $this->assertEquals([
       'contact_id' => 'contact_id varchar(255)',
-      'billing_city' => 'billing_city text',
-      'billing_street_address' => 'billing_street_address text',
-      'billing_postal_code' => 'billing_postal_code text',
-      'home_city' => 'home_city text',
-      'home_street_address' => 'home_street_address text',
-      'home_postal_code' => 'home_postal_code text',
-      'main_city' => 'main_city text',
-      'main_street_address' => 'main_street_address text',
-      'main_postal_code' => 'main_postal_code text',
-      'other_city' => 'other_city text',
-      'other_street_address' => 'other_street_address text',
-      'other_postal_code' => 'other_postal_code text',
-      'whare_kai_city' => 'whare_kai_city text',
-      'whare_kai_street_address' => 'whare_kai_street_address text',
-      'whare_kai_postal_code' => 'whare_kai_postal_code text',
-      '2_a_b_billing_city' => '2_a_b_billing_city text',
-      '2_a_b_home_city' => '2_a_b_home_city text',
-      '2_a_b_main_city' => '2_a_b_main_city text',
-      '2_a_b_other_city' => '2_a_b_other_city text',
-      '2_a_b_whare_kai_city' => '2_a_b_whare_kai_city text',
-      '2_a_b_city' => '2_a_b_city text',
-      '8_a_b_billing_city' => '8_a_b_billing_city text',
-      '8_a_b_home_city' => '8_a_b_home_city text',
-      '8_a_b_main_city' => '8_a_b_main_city text',
-      '8_a_b_other_city' => '8_a_b_other_city text',
-      '8_a_b_whare_kai_city' => '8_a_b_whare_kai_city text',
-      '8_a_b_city' => '8_a_b_city text',
-      '5_a_b_billing_city' => '5_a_b_billing_city text',
-      '5_a_b_home_city' => '5_a_b_home_city text',
-      '5_a_b_main_city' => '5_a_b_main_city text',
-      '5_a_b_other_city' => '5_a_b_other_city text',
-      '5_a_b_whare_kai_city' => '5_a_b_whare_kai_city text',
-      '5_a_b_city' => '5_a_b_city text',
+      'billing_city' => 'billing_city varchar(64)',
+      'billing_street_address' => 'billing_street_address varchar(96)',
+      'billing_postal_code' => 'billing_postal_code varchar(64)',
+      'home_city' => 'home_city varchar(64)',
+      'home_street_address' => 'home_street_address varchar(96)',
+      'home_postal_code' => 'home_postal_code varchar(64)',
+      'main_city' => 'main_city varchar(64)',
+      'main_street_address' => 'main_street_address varchar(96)',
+      'main_postal_code' => 'main_postal_code varchar(64)',
+      'other_city' => 'other_city varchar(64)',
+      'other_street_address' => 'other_street_address varchar(96)',
+      'other_postal_code' => 'other_postal_code varchar(64)',
+      'whare_kai_city' => 'whare_kai_city varchar(64)',
+      'whare_kai_street_address' => 'whare_kai_street_address varchar(96)',
+      'whare_kai_postal_code' => 'whare_kai_postal_code varchar(64)',
+      '2_a_b_billing_city' => '2_a_b_billing_city varchar(64)',
+      '2_a_b_home_city' => '2_a_b_home_city varchar(64)',
+      '2_a_b_main_city' => '2_a_b_main_city varchar(64)',
+      '2_a_b_other_city' => '2_a_b_other_city varchar(64)',
+      '2_a_b_whare_kai_city' => '2_a_b_whare_kai_city varchar(64)',
+      '2_a_b_city' => '2_a_b_city varchar(64)',
+      '8_a_b_billing_city' => '8_a_b_billing_city varchar(64)',
+      '8_a_b_home_city' => '8_a_b_home_city varchar(64)',
+      '8_a_b_main_city' => '8_a_b_main_city varchar(64)',
+      '8_a_b_other_city' => '8_a_b_other_city varchar(64)',
+      '8_a_b_whare_kai_city' => '8_a_b_whare_kai_city varchar(64)',
+      '8_a_b_city' => '8_a_b_city varchar(64)',
+      '5_a_b_billing_city' => '5_a_b_billing_city varchar(64)',
+      '5_a_b_home_city' => '5_a_b_home_city varchar(64)',
+      '5_a_b_main_city' => '5_a_b_main_city varchar(64)',
+      '5_a_b_other_city' => '5_a_b_other_city varchar(64)',
+      '5_a_b_whare_kai_city' => '5_a_b_whare_kai_city varchar(64)',
+      '5_a_b_city' => '5_a_b_city varchar(64)',
     ], $sqlColumns);
   }
 
@@ -2307,7 +2307,7 @@ class CRM_Export_BAO_ExportTest extends CiviUnitTestCase {
       'signature_text' => 'signature_text longtext',
       'signature_html' => 'signature_html longtext',
       'im_provider' => 'im_provider text',
-      'im' => 'im varchar(64)',
+      'im_screen_name' => 'im_screen_name varchar(64)',
       'openid' => 'openid varchar(255)',
       'world_region' => 'world_region varchar(128)',
       'url' => 'url varchar(128)',
@@ -2492,7 +2492,7 @@ class CRM_Export_BAO_ExportTest extends CiviUnitTestCase {
       'signature_text' => 'signature_text longtext',
       'signature_html' => 'signature_html longtext',
       'im_provider' => 'im_provider text',
-      'im' => 'im varchar(64)',
+      'im_screen_name' => 'im_screen_name varchar(64)',
       'openid' => 'openid varchar(255)',
       'world_region' => 'world_region varchar(128)',
       'url' => 'url varchar(128)',