From fa5bb5cf6db9d297b63db22a4c4a2361c8361cfe Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Fri, 6 Jun 2014 18:23:24 -0700 Subject: [PATCH] CRM-14811 - Add CRM_Core_InnoDBIndexer --- CRM/Core/InnoDBIndexer.php | 252 +++++++++++++++++++ tests/phpunit/CRM/Core/InnoDBIndexerTest.php | 83 ++++++ 2 files changed, 335 insertions(+) create mode 100644 CRM/Core/InnoDBIndexer.php create mode 100644 tests/phpunit/CRM/Core/InnoDBIndexerTest.php diff --git a/CRM/Core/InnoDBIndexer.php b/CRM/Core/InnoDBIndexer.php new file mode 100644 index 0000000000..a505feec53 --- /dev/null +++ b/CRM/Core/InnoDBIndexer.php @@ -0,0 +1,252 @@ + array( + array('street_address', 'city', 'postal_code') + ), + 'civicrm_contact' => array( + array('sort_name', 'nick_name', 'display_name'), + ), + 'civicrm_email' => array( + array('email') + ), + 'civicrm_note' => array( + array('subject', 'note'), + ), + 'civicrm_phone' => array( + array('phone'), + ), + 'civicrm_tag' => array( + array('name'), + ), + ); + self::$singleton = new self(TRUE, $indices); + } + return self::$singleton; + } + + /** + * (Setting Callback) + * Respond to changes in the "enable_innodb_fts" setting + * + * @param bool $oldValue + * @param bool $newValue + * @param array $metadata Specification of the setting (per *.settings.php) + */ + public static function onToggleFts($oldValue, $newValue, $metadata) { + $indexer = CRM_Core_InnoDBIndexer::singleton(); + $indexer->setActive($newValue); + $indexer->fixSchemaDifferences(); + } + + /** + * @var array (string $table => array $indices) + * + * ex: $indices['civicrm_contact'][0] = array('first_name', 'last_name'); + */ + protected $indices; + + /** + * @var bool + */ + protected $isActive; + + public function __construct($isActive, $indices) { + $this->isActive = $isActive; + $this->indices = $this->normalizeIndices($indices); + } + + public function fixSchemaDifferences() { + // Limitation: This won't pick up stale indices on tables which are not + // declared in $this->indices. That's not much of an issue for now b/c + // we have a static list of tables. + foreach ($this->indices as $tableName => $ign) { + $todoSqls = $this->reconcileIndexSqls($tableName); + foreach ($todoSqls as $todoSql) { + CRM_Core_DAO::executeQuery($todoSql); + } + } + } + + /** + * Determine if an index is expected to exist + * + * @param string $table + * @param array $fields list of field names that must be in the index + * @return bool + */ + public function hasDeclaredIndex($table, $fields) { + if (!$this->isActive) { + return FALSE; + } + + if (isset($this->indices[$table])) { + foreach ($this->indices[$table] as $idxFields) { + // TODO determine if $idxFields must be exact match or merely a subset + // if (sort($fields) == sort($idxFields)) { + if (array_diff($fields, $idxFields) == array()) { + return TRUE; + } + } + } + + return FALSE; + } + + /** + * Get a list of FTS index names that are currently defined in the database. + * + * @param string $table + * @return array (string $indexName => string $indexName) + */ + public function findActualFtsIndexNames($table) { + // Note: this only works in MySQL 5.6, but this whole system is intended to only work in MySQL 5.6 + $sql = " + SELECT i.name as index_name + FROM information_schema.innodb_sys_tables t + JOIN information_schema.innodb_sys_indexes i USING (table_id) + WHERE t.name = concat(database(),'/$table') + AND i.name like '" . self::IDX_PREFIX . "%' + "; + $dao = CRM_Core_DAO::executeQuery($sql); + $indexNames = array(); + while ($dao->fetch()) { + $indexNames[$dao->index_name] = $dao->index_name; + } + return $indexNames; + } + + /** + * Generate a "CREATE INDEX" statement for each desired + * FTS index. + * + * @param $table + * @return array (string $indexName => string $sql) + */ + public function buildIndexSql($table) { + $sqls = array(); // array (string $idxName => string $sql) + if ($this->isActive && isset($this->indices[$table])) { + foreach ($this->indices[$table] as $fields) { + $name = self::IDX_PREFIX . md5($table . '::' . implode(',', $fields)); + $sqls[$name] = sprintf("CREATE FULLTEXT INDEX %s ON %s (%s)", $name, $table, implode(',', $fields)); + } + } + return $sqls; + } + + /** + * Generate a "DROP INDEX" statement for each existing FTS index + * + * @param string $table + * @return array (string $idxName => string $sql) + */ + public function dropIndexSql($table) { + $sqls = array(); + $names = $this->findActualFtsIndexNames($table); + foreach ($names as $name) { + $sqls[$name] = sprintf("DROP INDEX %s ON %s", $name, $table); + } + return $sqls; + } + + /** + * Construct a set of SQL statements which will create (or preserve) + * required indices and destroy unneeded indices. + * + * @param $table + * @return array + */ + public function reconcileIndexSqls($table) { + $buildIndexSqls = $this->buildIndexSql($table); + $dropIndexSqls = $this->dropIndexSql($table); + + $allIndexNames = array_unique(array_merge( + array_keys($dropIndexSqls), + array_keys($buildIndexSqls) + )); + + $todoSqls = array(); + foreach ($allIndexNames as $indexName) { + if (isset($buildIndexSqls[$indexName]) && isset($dropIndexSqls[$indexName])) { + // already exists + } + elseif (isset($buildIndexSqls[$indexName])) { + $todoSqls[] = $buildIndexSqls[$indexName]; + } + else { + $todoSqls[] = $dropIndexSqls[$indexName]; + } + } + return $todoSqls; + } + + /** + * Put the indices into a normalized format + * + * @param $indices + * @return array + */ + public function normalizeIndices($indices) { + $result = array(); + foreach ($indices as $table => $indicesByTable) { + foreach ($indicesByTable as $k => $fields) { + sort($fields); + $result[$table][] = $fields; + } + } + return $result; + } + + /** + * @param boolean $isActive + */ + public function setActive($isActive) { + $this->isActive = $isActive; + } + + /** + * @return boolean + */ + public function getActive() { + return $this->isActive; + } +} diff --git a/tests/phpunit/CRM/Core/InnoDBIndexerTest.php b/tests/phpunit/CRM/Core/InnoDBIndexerTest.php new file mode 100644 index 0000000000..4d340a2e6e --- /dev/null +++ b/tests/phpunit/CRM/Core/InnoDBIndexerTest.php @@ -0,0 +1,83 @@ +fixSchemaDifferences(); + + parent::tearDown(); + } + + function testHasDeclaredIndex() { + $idx = new CRM_Core_InnoDBIndexer(TRUE, array( + 'civicrm_contact' => array( + array('first_name', 'last_name'), + array('foo') + ), + 'civicrm_email' => array( + array('whiz'), + ), + )); + + $this->assertTrue($idx->hasDeclaredIndex('civicrm_contact', array('first_name', 'last_name'))); + $this->assertTrue($idx->hasDeclaredIndex('civicrm_contact', array('last_name', 'first_name'))); + $this->assertTrue($idx->hasDeclaredIndex('civicrm_contact', array('first_name'))); // not sure if this is right behavior + $this->assertTrue($idx->hasDeclaredIndex('civicrm_contact', array('last_name'))); // not sure if this is right behavior + $this->assertTrue($idx->hasDeclaredIndex('civicrm_contact', array('foo'))); + $this->assertFalse($idx->hasDeclaredIndex('civicrm_contact', array('whiz'))); + + $this->assertFalse($idx->hasDeclaredIndex('civicrm_email', array('first_name', 'last_name'))); + $this->assertFalse($idx->hasDeclaredIndex('civicrm_email', array('foo'))); + $this->assertTrue($idx->hasDeclaredIndex('civicrm_email', array('whiz'))); + } + + /** + * When disabled, there is no FTS index, so queries that rely on FTS index fail. + */ + function testDisabled() { + $idx = new CRM_Core_InnoDBIndexer(FALSE, array( + 'civicrm_contact' => array( + array('first_name', 'last_name'), + ), + )); + $idx->fixSchemaDifferences(); + + try { + CRM_Core_DAO::executeQuery('SELECT id FROM civicrm_contact WHERE MATCH(first_name,last_name) AGAINST ("joe")'); + $this->fail("Missed expected exception"); + } catch (Exception $e) { + $this->assertTrue(TRUE, 'Received expected exception'); + } + } + + /** + * When enabled, the FTS index is created, so queries that rely on FTS work. + */ + function testEnabled() { + if (!$this->supportsFts()) { + $this->markTestSkipped("Local installation of InnoDB does not support FTS."); + return; + } + + $idx = new CRM_Core_InnoDBIndexer(TRUE, array( + 'civicrm_contact' => array( + array('first_name', 'last_name'), + ), + )); + $idx->fixSchemaDifferences(); + + CRM_Core_DAO::executeQuery('SELECT id FROM civicrm_contact WHERE MATCH(first_name,last_name) AGAINST ("joe")'); + } + + function supportsFts() { + return version_compare(CRM_Core_DAO::singleValueQuery('SELECT VERSION()'), '5.6.0', '>='); + } +} -- 2.25.1