tools/scripts/solr/createSolrJSON.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007.                                       |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License along with this program; if not, contact CiviCRM LLC       |
  21  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  22  | GNU Affero General Public License or the licensing of CiviCRM,     |
  23  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  24  +--------------------------------------------------------------------+
  25 */
  26
  27 /**
  28  * Create a xml file for a set of contact ID's in a format digestible
  29  * by Solr
  30  */
  31
  32 require_once '../../../civicrm.settings.php';
  33 require_once 'CRM/Core/Config.php';
  34
  35 define('CHUNK_SIZE', 128);
  36
  37 /**
  38  * Split a large array of contactIDs into more manageable smaller chunks
  39  * @param $contactIDs
  40  * @return array
  41  */
  42 function &splitContactIDs(&$contactIDs) {
  43   // contactIDs could be a real large array, so we split it up into
  44   // smaller chunks and then general xml for each chunk
  45   $chunks           = array();
  46   $current          = 0;
  47   $chunks[$current] = array();
  48   $count            = 0;
  49
  50   foreach ($contactIDs as $cid) {
  51     $chunks[$current][] = $cid;
  52     $count++;
  53
  54     if ($count == CHUNK_SIZE) {
  55       $current++;
  56       $chunks[$current] = array();
  57       $count = 0;
  58     }
  59   }
  60
  61   if (empty($chunks[$current])) {
  62     unset($chunks[$current]);
  63   }
  64
  65   return $chunks;
  66 }
  67
  68 /**
  69  * Given an array of values, generate the JSON in the Solr format
  70  * @param $values
  71  * @return string
  72  */
  73 function &generateSolrJSON($values) {
  74   $result = "[";
  75   foreach ($values as $cid => $tokens) {
  76     if (empty($tokens)) {
  77       continue;
  78     }
  79
  80     $result .= "\n  {\n    \"contact_id\" : \"$cid\",";
  81
  82     foreach ($tokens as $n => $v) {
  83       if (is_array($v)) {
  84         $str = array();
  85         foreach ($v as $el) {
  86           $el = escapeJsonString($el);
  87           $str[] = "\"$el\"";
  88         }
  89         $string = implode(",", $str);
  90         $result .= "\n    \"{$n}\" : [$string],";
  91       }
  92       else {
  93         $v = escapeJsonString($v);
  94         $result .= "\n    \"{$n}\" : \"{$v}\",";
  95       }
  96     }
  97
  98     // remove the last comma
  99     $result = rtrim($result, ",");
 100
 101     $result .= "\n  },";
 102   }
 103   // remove the last comma
 104   $result = rtrim($result, ",");
 105
 106   $result .= "\n]\n";
 107
 108
 109   return $result;
 110 }
 111
 112 /**
 113  * @param $value
 114  *
 115  * @return mixed
 116  */
 117 function escapeJsonString($value) {
 118   $escapers = array("\\", "/", "\"", "\n", "\r", "\t", "\x08", "\x0c");
 119   $replacements = array("\\\\", "\\/", "\\\"", "\\n", "\\r", "\\t", "\\f", "\\b");
 120   return str_replace($escapers, $replacements, $value);
 121 }
 122
 123 /**
 124  * Given a set of contact IDs get the values
 125  * @param $contactIDs
 126  * @param $values
 127  * @return array
 128  */
 129 function getValues(&$contactIDs, &$values) {
 130   $values = array();
 131
 132   foreach ($contactIDs as $cid) {
 133     $values[$cid] = array();
 134   }
 135
 136   getContactInfo($contactIDs, $values);
 137   getAddressInfo($contactIDs, $values);
 138   getPhoneInfo($contactIDs, $values);
 139   getEmailInfo($contactIDs, $values);
 140   getNoteInfo($contactIDs, $values);
 141
 142   return $values;
 143 }
 144
 145 /**
 146  * @param $contactIDs
 147  * @param $values
 148  * @param $tableName
 149  * @param $fields
 150  * @param $whereField
 151  * @param null $additionalWhereCond
 152  */
 153 function getTableInfo(&$contactIDs, &$values, $tableName, &$fields, $whereField, $additionalWhereCond = NULL) {
 154   $selectString = implode(',', array_keys($fields));
 155   $idString = implode(',', $contactIDs);
 156
 157   $sql = "
 158 SELECT $selectString, $whereField as contact_id
 159   FROM $tableName
 160  WHERE $whereField IN ( $idString )
 161 ";
 162
 163   if ($additionalWhereCond) {
 164     $sql .= " AND $additionalWhereCond";
 165   }
 166
 167   $dao = &CRM_Core_DAO::executeQuery($sql);
 168   while ($dao->fetch()) {
 169     foreach ($fields as $fld => $name) {
 170       $name = $name ? $name : $fld;
 171       appendValue($values, $dao->contact_id, $name, $dao->$fld);
 172     }
 173   }
 174 }
 175
 176 /**
 177  * @param $contactIDs
 178  * @param $values
 179  */
 180 function getContactInfo(&$contactIDs, &$values) {
 181   $fields = array('sort_name' => NULL,
 182     'display_name' => NULL,
 183     'contact_type' => NULL,
 184     'legal_identifier' => NULL,
 185     'external_identifier' => NULL,
 186     'first_name' => NULL,
 187     'last_name' => NULL,
 188     'middle_name' => NULL,
 189     'household_name' => NULL,
 190     'organization_name' => NULL,
 191     'legal_name' => NULL,
 192     'job_title' => NULL,
 193   );
 194   getTableInfo($contactIDs, $values, 'civicrm_contact', $fields, 'id');
 195 }
 196
 197 /**
 198  * @param $contactIDs
 199  * @param $values
 200  */
 201 function getNoteInfo(&$contactIDs, &$values) {
 202   $ids = implode(',', $contactIDs);
 203
 204   $sql = "
 205 SELECT
 206       entity_id as contact_id,
 207       note as note, subject as subject
 208 FROM  civicrm_note
 209 WHERE entity_id IN ( $ids )
 210 AND   entity_table = 'civicrm_contact'
 211 ";
 212
 213   $dao = &CRM_Core_DAO::executeQuery($sql);
 214   while ($dao->fetch()) {
 215     $note = empty($dao->subject) ? '' : "{$dao->subject}: ";
 216     $note .= empty($dao->note) ? '' : $dao->note;
 217
 218     appendValue($values, $dao->contact_id, 'note', $note);
 219   }
 220 }
 221
 222 /**
 223  * @param $contactIDs
 224  * @param $values
 225  */
 226 function getPhoneInfo(&$contactIDs, &$values) {
 227   $ids = implode(',', $contactIDs);
 228
 229   $sql = "
 230 SELECT
 231   c.id as contact_id,
 232   l.name as location_type,
 233   p.phone as phone,
 234   v.label as phone_type
 235 FROM      civicrm_contact c
 236 INNER JOIN civicrm_phone          p  ON p.contact_id        = c.id
 237 LEFT  JOIN civicrm_location_type  l  ON p.location_type_id  = l.id
 238 LEFT  JOIN civicrm_option_group   g  ON g.name = 'phone_type'
 239 LEFT  JOIN civicrm_option_value   v  ON v.option_group_id = g.id AND p.phone_type_id = v.value
 240 WHERE      c.id IN ( $ids )
 241 AND        p.phone IS NOT NULL
 242 ";
 243
 244   $dao = &CRM_Core_DAO::executeQuery($sql);
 245   while ($dao->fetch()) {
 246     $phone = '';
 247
 248     if (!empty($dao->location_type)) {
 249       $phone = "{$dao->location_type}: ";
 250     }
 251
 252     $phone .= $dao->phone;
 253
 254     if (!empty($dao->phone_type)) {
 255       $phone .= " ({$dao->phone_type})";
 256     }
 257
 258     appendValue($values, $dao->contact_id, 'phone', $phone);
 259   }
 260 }
 261
 262 /**
 263  * @param $contactIDs
 264  * @param $values
 265  */
 266 function getEmailInfo(&$contactIDs, &$values) {
 267   $ids = implode(',', $contactIDs);
 268
 269   $sql = "
 270 SELECT
 271   c.id as contact_id,
 272   l.name as location_type,
 273   e.email as email
 274 FROM      civicrm_contact c
 275 INNER JOIN civicrm_email          e  ON e.contact_id        = c.id
 276 LEFT  JOIN civicrm_location_type  l  ON e.location_type_id  = l.id
 277 WHERE      c.id IN ( $ids )
 278 AND        e.email IS NOT NULL
 279 ";
 280
 281   $dao = &CRM_Core_DAO::executeQuery($sql);
 282   while ($dao->fetch()) {
 283     $email = '';
 284
 285     if (!empty($dao->location_type)) {
 286       $email = "{$dao->location_type}: ";
 287     }
 288
 289     $email .= $dao->email;
 290     appendValue($values, $dao->contact_id, 'email', $email);
 291   }
 292 }
 293
 294 /**
 295  * @param $contactIDs
 296  * @param $values
 297  */
 298 function getAddressInfo(&$contactIDs, &$values) {
 299   $ids = implode(',', $contactIDs);
 300
 301   $sql = "
 302 SELECT     c.id as contact_id, l.name as location_type,
 303            a.street_address, a.supplemental_address_1, a.supplemental_address_2,
 304            a.supplemental_address_3,
 305            a.city, a.postal_code,
 306            s.name as state, co.name as country
 307 FROM       civicrm_contact c
 308 INNER JOIN civicrm_address        a  ON a.contact_id        = c.id
 309 LEFT  JOIN civicrm_location_type  l  ON a.location_type_id  = l.id
 310 LEFT  JOIN civicrm_state_province s  ON a.state_province_id = s.id
 311 LEFT  JOIN civicrm_country        co ON a.country_id        = co.id
 312 WHERE c.id IN ( $ids )
 313 ";
 314
 315   $fields = array('location_type', 'street_address', 'supplemental_address_1',
 316     'supplemental_address_2', 'supplemental_address_3', 'city', 'postal_code',
 317     'state', 'country',
 318   );
 319   $dao = &CRM_Core_DAO::executeQuery($sql);
 320   while ($dao->fetch()) {
 321     $address = '';
 322     foreach ($fields as $fld) {
 323       if (empty($dao->$fld)) {
 324         continue;
 325       }
 326
 327       $address .= ($fld == 'location_type') ? "{$dao->$fld}: " : " {$dao->$fld},";
 328       appendValue($values, $dao->contact_id, $fld, $dao->$fld);
 329     }
 330
 331     if (!empty($address)) {
 332       $address = rtrim($address, ",");
 333       appendValue($values, $dao->contact_id, 'address', $address);
 334     }
 335   }
 336 }
 337
 338 /**
 339  * @param $values
 340  * @param $contactID
 341  * @param $name
 342  * @param $value
 343  */
 344 function appendValue(&$values, $contactID, $name, $value) {
 345   if (empty($value)) {
 346     return;
 347   }
 348
 349   if (!isset($values[$contactID][$name])) {
 350     $values[$contactID][$name] = $value;
 351   }
 352   else {
 353     if (!is_array($values[$contactID][$name])) {
 354       $save = $values[$contactID][$name];
 355       $values[$contactID][$name] = array();
 356       $values[$contactID][$name][] = $save;
 357     }
 358     $values[$contactID][$name][] = $value;
 359   }
 360 }
 361
 362 /**
 363  * @param $contactIDs
 364  */
 365 function run(&$contactIDs) {
 366   $chunks = &splitContactIDs($contactIDs);
 367
 368   foreach ($chunks as $chunk) {
 369     $values = array();
 370     getValues($chunk, $values);
 371     $xml = &generateSolrJSON($values);
 372     echo $xml;
 373   }
 374 }
 375
 376 $config = &CRM_Core_Config::singleton();
 377 $config->userFramework = 'Soap';
 378 $config->userFrameworkClass = 'CRM_Utils_System_Soap';
 379 $config->userHookClass = 'CRM_Utils_Hook_Soap';
 380
 381 $sql = <<<EOT
 382 SELECT id
 383 FROM civicrm_contact
 384 EOT;
 385 $dao = &CRM_Core_DAO::executeQuery($sql);
 386
 387
 388 $contactIDs = array();
 389 while ($dao->fetch()) {
 390   $contactIDs[] = $dao->id;
 391 }
 392
 393 run($contactIDs);
 394