tools/scripts/solr/createSolrJSON.php

   1 <?php
   2 /*
   3  +--------------------------------------------------------------------+
   4  | CiviCRM version 5                                                  |
   5  +--------------------------------------------------------------------+
   6  | Copyright CiviCRM LLC (c) 2004-2019                                |
   7  +--------------------------------------------------------------------+
   8  | This file is a part of CiviCRM.                                    |
   9  |                                                                    |
  10  | CiviCRM is free software; you can copy, modify, and distribute it  |
  11  | under the terms of the GNU Affero General Public License           |
  12  | Version 3, 19 November 2007.                                       |
  13  |                                                                    |
  14  | CiviCRM is distributed in the hope that it will be useful, but     |
  15  | WITHOUT ANY WARRANTY; without even the implied warranty of         |
  16  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.               |
  17  | See the GNU Affero General Public License for more details.        |
  18  |                                                                    |
  19  | You should have received a copy of the GNU Affero General Public   |
  20  | License along with this program; if not, contact CiviCRM LLC       |
  21  | at info[AT]civicrm[DOT]org. If you have questions about the        |
  22  | GNU Affero General Public License or the licensing of CiviCRM,     |
  23  | see the CiviCRM license FAQ at http://civicrm.org/licensing        |
  24  +--------------------------------------------------------------------+
  25 */
  26
  27 /**
  28  * Create a xml file for a set of contact ID's in a format digestible
  29  * by Solr
  30  */
  31
  32 require_once '../../../civicrm.settings.php';
  33 require_once 'CRM/Core/Config.php';
  34
  35 define('CHUNK_SIZE', 128);
  36
  37 /**
  38  * Split a large array of contactIDs into more manageable smaller chunks
  39  * @param $contactIDs
  40  * @return array
  41  */
  42 function &splitContactIDs(&$contactIDs) {
  43   // contactIDs could be a real large array, so we split it up into
  44   // smaller chunks and then general xml for each chunk
  45   $chunks = [];
  46   $current = 0;
  47   $chunks[$current] = [];
  48   $count = 0;
  49
  50   foreach ($contactIDs as $cid) {
  51     $chunks[$current][] = $cid;
  52     $count++;
  53
  54     if ($count == CHUNK_SIZE) {
  55       $current++;
  56       $chunks[$current] = [];
  57       $count = 0;
  58     }
  59   }
  60
  61   if (empty($chunks[$current])) {
  62     unset($chunks[$current]);
  63   }
  64
  65   return $chunks;
  66 }
  67
  68 /**
  69  * Given an array of values, generate the JSON in the Solr format
  70  * @param $values
  71  * @return string
  72  */
  73 function &generateSolrJSON($values) {
  74   $result = "[";
  75   foreach ($values as $cid => $tokens) {
  76     if (empty($tokens)) {
  77       continue;
  78     }
  79
  80     $result .= "\n  {\n    \"contact_id\" : \"$cid\",";
  81
  82     foreach ($tokens as $n => $v) {
  83       if (is_array($v)) {
  84         $str = [];
  85         foreach ($v as $el) {
  86           $el = escapeJsonString($el);
  87           $str[] = "\"$el\"";
  88         }
  89         $string = implode(",", $str);
  90         $result .= "\n    \"{$n}\" : [$string],";
  91       }
  92       else {
  93         $v = escapeJsonString($v);
  94         $result .= "\n    \"{$n}\" : \"{$v}\",";
  95       }
  96     }
  97
  98     // remove the last comma
  99     $result = rtrim($result, ",");
 100
 101     $result .= "\n  },";
 102   }
 103   // remove the last comma
 104   $result = rtrim($result, ",");
 105
 106   $result .= "\n]\n";
 107
 108
 109   return $result;
 110 }
 111
 112 /**
 113  * @param $value
 114  *
 115  * @return mixed
 116  */
 117 function escapeJsonString($value) {
 118   $escapers = ["\\", "/", "\"", "\n", "\r", "\t", "\x08", "\x0c"];
 119   $replacements = ["\\\\", "\\/", "\\\"", "\\n", "\\r", "\\t", "\\f", "\\b"];
 120   return str_replace($escapers, $replacements, $value);
 121 }
 122
 123 /**
 124  * Given a set of contact IDs get the values
 125  * @param $contactIDs
 126  * @param $values
 127  * @return array
 128  */
 129 function getValues(&$contactIDs, &$values) {
 130   $values = [];
 131
 132   foreach ($contactIDs as $cid) {
 133     $values[$cid] = [];
 134   }
 135
 136   getContactInfo($contactIDs, $values);
 137   getAddressInfo($contactIDs, $values);
 138   getPhoneInfo($contactIDs, $values);
 139   getEmailInfo($contactIDs, $values);
 140   getNoteInfo($contactIDs, $values);
 141
 142   return $values;
 143 }
 144
 145 /**
 146  * @param $contactIDs
 147  * @param $values
 148  * @param $tableName
 149  * @param $fields
 150  * @param $whereField
 151  * @param null $additionalWhereCond
 152  */
 153 function getTableInfo(&$contactIDs, &$values, $tableName, &$fields, $whereField, $additionalWhereCond = NULL) {
 154   $selectString = implode(',', array_keys($fields));
 155   $idString = implode(',', $contactIDs);
 156
 157   $sql = "
 158 SELECT $selectString, $whereField as contact_id
 159   FROM $tableName
 160  WHERE $whereField IN ( $idString )
 161 ";
 162
 163   if ($additionalWhereCond) {
 164     $sql .= " AND $additionalWhereCond";
 165   }
 166
 167   $dao = &CRM_Core_DAO::executeQuery($sql);
 168   while ($dao->fetch()) {
 169     foreach ($fields as $fld => $name) {
 170       $name = $name ? $name : $fld;
 171       appendValue($values, $dao->contact_id, $name, $dao->$fld);
 172     }
 173   }
 174 }
 175
 176 /**
 177  * @param $contactIDs
 178  * @param $values
 179  */
 180 function getContactInfo(&$contactIDs, &$values) {
 181   $fields = [
 182     'sort_name' => NULL,
 183     'display_name' => NULL,
 184     'contact_type' => NULL,
 185     'legal_identifier' => NULL,
 186     'external_identifier' => NULL,
 187     'first_name' => NULL,
 188     'last_name' => NULL,
 189     'middle_name' => NULL,
 190     'household_name' => NULL,
 191     'organization_name' => NULL,
 192     'legal_name' => NULL,
 193     'job_title' => NULL,
 194   ];
 195   getTableInfo($contactIDs, $values, 'civicrm_contact', $fields, 'id');
 196 }
 197
 198 /**
 199  * @param $contactIDs
 200  * @param $values
 201  */
 202 function getNoteInfo(&$contactIDs, &$values) {
 203   $ids = implode(',', $contactIDs);
 204
 205   $sql = "
 206 SELECT
 207       entity_id as contact_id,
 208       note as note, subject as subject
 209 FROM  civicrm_note
 210 WHERE entity_id IN ( $ids )
 211 AND   entity_table = 'civicrm_contact'
 212 ";
 213
 214   $dao = &CRM_Core_DAO::executeQuery($sql);
 215   while ($dao->fetch()) {
 216     $note = empty($dao->subject) ? '' : "{$dao->subject}: ";
 217     $note .= empty($dao->note) ? '' : $dao->note;
 218
 219     appendValue($values, $dao->contact_id, 'note', $note);
 220   }
 221 }
 222
 223 /**
 224  * @param $contactIDs
 225  * @param $values
 226  */
 227 function getPhoneInfo(&$contactIDs, &$values) {
 228   $ids = implode(',', $contactIDs);
 229
 230   $sql = "
 231 SELECT
 232   c.id as contact_id,
 233   l.name as location_type,
 234   p.phone as phone,
 235   v.label as phone_type
 236 FROM      civicrm_contact c
 237 INNER JOIN civicrm_phone          p  ON p.contact_id        = c.id
 238 LEFT  JOIN civicrm_location_type  l  ON p.location_type_id  = l.id
 239 LEFT  JOIN civicrm_option_group   g  ON g.name = 'phone_type'
 240 LEFT  JOIN civicrm_option_value   v  ON v.option_group_id = g.id AND p.phone_type_id = v.value
 241 WHERE      c.id IN ( $ids )
 242 AND        p.phone IS NOT NULL
 243 ";
 244
 245   $dao = &CRM_Core_DAO::executeQuery($sql);
 246   while ($dao->fetch()) {
 247     $phone = '';
 248
 249     if (!empty($dao->location_type)) {
 250       $phone = "{$dao->location_type}: ";
 251     }
 252
 253     $phone .= $dao->phone;
 254
 255     if (!empty($dao->phone_type)) {
 256       $phone .= " ({$dao->phone_type})";
 257     }
 258
 259     appendValue($values, $dao->contact_id, 'phone', $phone);
 260   }
 261 }
 262
 263 /**
 264  * @param $contactIDs
 265  * @param $values
 266  */
 267 function getEmailInfo(&$contactIDs, &$values) {
 268   $ids = implode(',', $contactIDs);
 269
 270   $sql = "
 271 SELECT
 272   c.id as contact_id,
 273   l.name as location_type,
 274   e.email as email
 275 FROM      civicrm_contact c
 276 INNER JOIN civicrm_email          e  ON e.contact_id        = c.id
 277 LEFT  JOIN civicrm_location_type  l  ON e.location_type_id  = l.id
 278 WHERE      c.id IN ( $ids )
 279 AND        e.email IS NOT NULL
 280 ";
 281
 282   $dao = &CRM_Core_DAO::executeQuery($sql);
 283   while ($dao->fetch()) {
 284     $email = '';
 285
 286     if (!empty($dao->location_type)) {
 287       $email = "{$dao->location_type}: ";
 288     }
 289
 290     $email .= $dao->email;
 291     appendValue($values, $dao->contact_id, 'email', $email);
 292   }
 293 }
 294
 295 /**
 296  * @param $contactIDs
 297  * @param $values
 298  */
 299 function getAddressInfo(&$contactIDs, &$values) {
 300   $ids = implode(',', $contactIDs);
 301
 302   $sql = "
 303 SELECT     c.id as contact_id, l.name as location_type,
 304            a.street_address, a.supplemental_address_1, a.supplemental_address_2,
 305            a.supplemental_address_3,
 306            a.city, a.postal_code,
 307            s.name as state, co.name as country
 308 FROM       civicrm_contact c
 309 INNER JOIN civicrm_address        a  ON a.contact_id        = c.id
 310 LEFT  JOIN civicrm_location_type  l  ON a.location_type_id  = l.id
 311 LEFT  JOIN civicrm_state_province s  ON a.state_province_id = s.id
 312 LEFT  JOIN civicrm_country        co ON a.country_id        = co.id
 313 WHERE c.id IN ( $ids )
 314 ";
 315
 316   $fields = [
 317     'location_type',
 318     'street_address',
 319     'supplemental_address_1',
 320     'supplemental_address_2',
 321     'supplemental_address_3',
 322     'city',
 323     'postal_code',
 324     'state',
 325     'country',
 326   ];
 327   $dao = &CRM_Core_DAO::executeQuery($sql);
 328   while ($dao->fetch()) {
 329     $address = '';
 330     foreach ($fields as $fld) {
 331       if (empty($dao->$fld)) {
 332         continue;
 333       }
 334
 335       $address .= ($fld == 'location_type') ? "{$dao->$fld}: " : " {$dao->$fld},";
 336       appendValue($values, $dao->contact_id, $fld, $dao->$fld);
 337     }
 338
 339     if (!empty($address)) {
 340       $address = rtrim($address, ",");
 341       appendValue($values, $dao->contact_id, 'address', $address);
 342     }
 343   }
 344 }
 345
 346 /**
 347  * @param $values
 348  * @param $contactID
 349  * @param $name
 350  * @param $value
 351  */
 352 function appendValue(&$values, $contactID, $name, $value) {
 353   if (empty($value)) {
 354     return;
 355   }
 356
 357   if (!isset($values[$contactID][$name])) {
 358     $values[$contactID][$name] = $value;
 359   }
 360   else {
 361     if (!is_array($values[$contactID][$name])) {
 362       $save = $values[$contactID][$name];
 363       $values[$contactID][$name] = [];
 364       $values[$contactID][$name][] = $save;
 365     }
 366     $values[$contactID][$name][] = $value;
 367   }
 368 }
 369
 370 /**
 371  * @param $contactIDs
 372  */
 373 function run(&$contactIDs) {
 374   $chunks = &splitContactIDs($contactIDs);
 375
 376   foreach ($chunks as $chunk) {
 377     $values = [];
 378     getValues($chunk, $values);
 379     $xml = &generateSolrJSON($values);
 380     echo $xml;
 381   }
 382 }
 383
 384 $config = CRM_Core_Config::singleton();
 385 $config->userFramework = 'Soap';
 386 $config->userFrameworkClass = 'CRM_Utils_System_Soap';
 387 $config->userHookClass = 'CRM_Utils_Hook_Soap';
 388
 389 $sql = <<<EOT
 390 SELECT id
 391 FROM civicrm_contact
 392 EOT;
 393 $dao = &CRM_Core_DAO::executeQuery($sql);
 394
 395
 396 $contactIDs = [];
 397 while ($dao->fetch()) {
 398   $contactIDs[] = $dao->id;
 399 }
 400
 401 run($contactIDs);
 402