tools/scripts/solr/createSolrXML.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007.                                       |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License along with this program; if not, contact CiviCRM LLC       |
  21  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  22  | GNU Affero General Public License or the licensing of CiviCRM,     |
  23  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  24  +--------------------------------------------------------------------+
  25 */
  26
  27 /**
  28  * Create a xml file for a set of contact ID's in a format digestible
  29  * by Solr
  30  */
  31
  32 require_once '../../civicrm.config.php';
  33 require_once 'CRM/Core/Config.php';
  34
  35 define('CHUNK_SIZE', 128);
  36
  37 /**
  38  * Split a large array of contactIDs into more manageable smaller chunks
  39  * @param $contactIDs
  40  * @return array
  41  */
  42 function &splitContactIDs(&$contactIDs) {
  43   // contactIDs could be a real large array, so we split it up into
  44   // smaller chunks and then general xml for each chunk
  45   $chunks = [];
  46   $current = 0;
  47   $chunks[$current] = [];
  48   $count = 0;
  49
  50   foreach ($contactIDs as $cid) {
  51     $chunks[$current][] = $cid;
  52     $count++;
  53
  54     if ($count == CHUNK_SIZE) {
  55       $current++;
  56       $chunks[$current] = [];
  57       $count = 0;
  58     }
  59   }
  60
  61   if (empty($chunks[$current])) {
  62     unset($chunks[$current]);
  63   }
  64
  65   return $chunks;
  66 }
  67
  68 /**
  69  * Given an array of values, generate the XML in the Solr format
  70  * @param $values
  71  * @return string
  72  */
  73 function &generateSolrXML($values) {
  74   $result = "<add>\n";
  75   foreach ($values as $cid => $tokens) {
  76     if (empty($tokens)) {
  77       continue;
  78     }
  79
  80     $result .= <<<EOT
  81   <doc>
  82     <field name="id">$cid</field>\n
  83 EOT;
  84
  85     foreach ($tokens as $t) {
  86       $result .= <<<EOT
  87     <field name="$t[0]">$t[1]</field>\n
  88 EOT;
  89     }
  90
  91     $result .= "  </doc>\n";
  92   }
  93   $result .= "</add>\n";
  94
  95
  96   return $result;
  97 }
  98
  99 /**
 100  * Given a set of contact IDs get the values
 101  * @param $contactIDs
 102  * @param $values
 103  * @return array
 104  */
 105 function getValues(&$contactIDs, &$values) {
 106   $values = [];
 107
 108   foreach ($contactIDs as $cid) {
 109     $values[$cid] = [];
 110   }
 111
 112   getContactInfo($contactIDs, $values);
 113   getLocationInfo($contactIDs, $values);
 114
 115   return $values;
 116 }
 117
 118 /**
 119  * @param $contactIDs
 120  * @param $values
 121  * @param $tableName
 122  * @param $fields
 123  * @param $whereField
 124  * @param null $additionalWhereCond
 125  */
 126 function getTableInfo(&$contactIDs, &$values, $tableName, &$fields, $whereField, $additionalWhereCond = NULL) {
 127   $selectString = implode(',', array_keys($fields));
 128   $idString = implode(',', $contactIDs);
 129
 130   $sql = "
 131 SELECT $selectString, $whereField as contact_id
 132   FROM $tableName
 133  WHERE $whereField IN ( $idString )
 134 ";
 135
 136   if ($additionalWhereCond) {
 137     $sql .= " AND $additionalWhereCond";
 138   }
 139
 140   $dao = CRM_Core_DAO::executeQuery($sql);
 141   while ($dao->fetch()) {
 142     foreach ($fields as $fld => $name) {
 143       if (empty($dao->$fld)) {
 144         continue;
 145       }
 146       if (!$name) {
 147         $name = $fld;
 148       }
 149       $values[$dao->contact_id][] = [$name, $dao->$fld];
 150     }
 151   }
 152 }
 153
 154 /**
 155  * @param $contactIDs
 156  * @param $values
 157  */
 158 function getContactInfo(&$contactIDs, &$values) {
 159   $fields = [
 160     'sort_name' => NULL,
 161     'display_name' => NULL,
 162     'contact_type' => NULL,
 163     'legal_identifier' => NULL,
 164     'external_identifier' => NULL,
 165     'source' => 'contact_source',
 166   ];
 167   getTableInfo($contactIDs, $values, 'civicrm_contact', $fields, 'id');
 168
 169   $fields = [
 170     'first_name' => NULL,
 171     'last_name' => NULL,
 172     'middle_name' => NULL,
 173     'job_title' => NULL,
 174   ];
 175   getTableInfo($contactIDs, $values, 'civicrm_individual', $fields, 'contact_id');
 176
 177   $fields = ['household_name' => NULL];
 178   getTableInfo($contactIDs, $values, 'civicrm_household', $fields, 'contact_id');
 179
 180   $fields = [
 181     'organization_name' => NULL,
 182     'legal_name' => NULL,
 183     'sic_code' => NULL,
 184   ];
 185   getTableInfo($contactIDs, $values, 'civicrm_organization', $fields, 'contact_id');
 186
 187   $fields = [
 188     'note' => 'note_body',
 189     'subject' => 'note_subject',
 190   ];
 191   getTableInfo($contactIDs, $values, 'civicrm_note', $fields, 'entity_id', "entity_table = 'civicrm_contact'");
 192 }
 193
 194 /**
 195  * @param $contactIDs
 196  * @param $values
 197  */
 198 function getLocationInfo(&$contactIDs, &$values) {
 199   $ids = implode(',', $contactIDs);
 200
 201   $sql = "
 202 SELECT
 203   l.entity_id as contact_id, l.name as location_name,
 204   a.street_address, a.supplemental_address_1, a.supplemental_address_2,
 205   a.supplemental_address_3,
 206   a.city, a.postal_code,
 207   co.name as county, s.name as state, c.name as country,
 208   e.email, p.phone, i.name as im
 209 FROM
 210   civicrm_location l
 211 LEFT JOIN civicrm_address        a  ON a.location_id       = l.id
 212 LEFT JOIN civicrm_email          e  ON e.location_id       = l.id
 213 LEFT JOIN civicrm_phone          p  ON p.location_id       = l.id
 214 LEFT JOIN civicrm_im             i  ON i.location_id       = l.id
 215 LEFT JOIN civicrm_state_province s  ON a.state_province_id = s.id
 216 LEFT JOIN civicrm_country        c  ON a.country_id        = c.id
 217 LEFT JOIN civicrm_county         co ON a.county_id         = co.id
 218 WHERE l.entity_table = 'civicrm_contact'
 219   AND l.entity_id IN ( $ids )
 220 ";
 221
 222   $fields = [
 223     'location_name',
 224     'street_address',
 225     'supplemental_address_1',
 226     'supplemental_address_2',
 227     'supplemental_address_3',
 228     'city',
 229     'postal_code',
 230     'county',
 231     'state',
 232     'country',
 233     'email',
 234     'phone',
 235     'im',
 236   ];
 237   $dao = CRM_Core_DAO::executeQuery($sql);
 238   while ($dao->fetch()) {
 239     foreach ($fields as $fld) {
 240       if (empty($dao->$fld)) {
 241         continue;
 242       }
 243       $values[$dao->contact_id][] = [$fld, $dao->$fld];
 244     }
 245   }
 246 }
 247
 248 /**
 249  * @param $contactIDs
 250  */
 251 function run(&$contactIDs) {
 252   $chunks = &splitContactIDs($contactIDs);
 253
 254   foreach ($chunks as $chunk) {
 255     $values = [];
 256     getValues($chunk, $values);
 257     $xml = &generateSolrXML($values);
 258     echo $xml;
 259   }
 260 }
 261
 262 $config = CRM_Core_Config::singleton();
 263 $config->userFramework = 'Soap';
 264 $config->userFrameworkClass = 'CRM_Utils_System_Soap';
 265 $config->userHookClass = 'CRM_Utils_Hook_Soap';
 266
 267 $sql = <<<EOT
 268 SELECT id
 269 FROM civicrm_contact
 270 EOT;
 271 $dao = CRM_Core_DAO::executeQuery($sql);
 272
 273 $contactIDs = [];
 274 while ($dao->fetch()) {
 275   $contactIDs[] = $dao->id;
 276 }
 277
 278 run($contactIDs);
 279