Commit | Line | Data |
---|---|---|
9d7974eb EM |
1 | <?php |
2 | /* | |
3 | +--------------------------------------------------------------------+ | |
4 | | Copyright CiviCRM LLC. All rights reserved. | | |
5 | | | | |
6 | | This work is published under the GNU AGPLv3 license with some | | |
7 | | permitted exceptions and without any warranty. For full license | | |
8 | | and copyright information, see https://civicrm.org/licensing | | |
9 | +--------------------------------------------------------------------+ | |
10 | */ | |
11 | ||
12 | /** | |
13 | * | |
14 | * @package CRM | |
15 | * @copyright CiviCRM LLC https://civicrm.org/licensing | |
16 | */ | |
17 | ||
7b057b66 | 18 | use Civi\Api4\UserJob; |
99e3c5f7 | 19 | use League\Csv\Writer; |
7b057b66 | 20 | |
9d7974eb EM |
21 | /** |
22 | * This class helps the forms within the import flow access submitted & parsed values. | |
23 | */ | |
24 | class CRM_Import_Forms extends CRM_Core_Form { | |
25 | ||
26 | /** | |
7b057b66 EM |
27 | * User job id. |
28 | * | |
29 | * This is the primary key of the civicrm_user_job table which is used to | |
30 | * track the import. | |
31 | * | |
32 | * @var int | |
33 | */ | |
34 | protected $userJobID; | |
35 | ||
36 | /** | |
37 | * @return int|null | |
38 | */ | |
39 | public function getUserJobID(): ?int { | |
40 | if (!$this->userJobID && $this->get('user_job_id')) { | |
41 | $this->userJobID = $this->get('user_job_id'); | |
42 | } | |
43 | return $this->userJobID; | |
44 | } | |
45 | ||
46 | /** | |
47 | * Set user job ID. | |
48 | * | |
49 | * @param int $userJobID | |
50 | */ | |
51 | public function setUserJobID(int $userJobID): void { | |
52 | $this->userJobID = $userJobID; | |
53 | // This set allows other forms in the flow ot use $this->get('user_job_id'). | |
54 | $this->set('user_job_id', $userJobID); | |
55 | } | |
56 | ||
57 | /** | |
58 | * User job details. | |
59 | * | |
60 | * This is the relevant row from civicrm_user_job. | |
61 | * | |
62 | * @var array | |
63 | */ | |
64 | protected $userJob; | |
65 | ||
83a1c234 EM |
66 | /** |
67 | * @var \CRM_Import_Parser | |
68 | */ | |
69 | protected $parser; | |
70 | ||
7b057b66 EM |
71 | /** |
72 | * Get User Job. | |
73 | * | |
74 | * API call to retrieve the userJob row. | |
75 | * | |
76 | * @return array | |
77 | * | |
78 | * @throws \API_Exception | |
79 | */ | |
80 | protected function getUserJob(): array { | |
81 | if (!$this->userJob) { | |
82 | $this->userJob = UserJob::get() | |
83 | ->addWhere('id', '=', $this->getUserJobID()) | |
84 | ->execute() | |
85 | ->first(); | |
86 | } | |
87 | return $this->userJob; | |
88 | } | |
89 | ||
90 | /** | |
91 | * Get submitted values stored in the user job. | |
92 | * | |
93 | * @return array | |
94 | * @throws \API_Exception | |
95 | */ | |
96 | protected function getUserJobSubmittedValues(): array { | |
97 | return $this->getUserJob()['metadata']['submitted_values']; | |
98 | } | |
99 | ||
100 | /** | |
101 | * Fields that may be submitted on any form in the flow. | |
102 | * | |
103 | * @var string[] | |
104 | */ | |
105 | protected $submittableFields = [ | |
106 | // Skip column header is actually a field that would be added from the | |
107 | // datasource - but currently only in contact, it is always there for | |
108 | // other imports, ditto uploadFile. | |
109 | 'skipColumnHeader' => 'DataSource', | |
110 | 'fieldSeparator' => 'DataSource', | |
111 | 'uploadFile' => 'DataSource', | |
112 | 'contactType' => 'DataSource', | |
dfa2f16c | 113 | 'contactSubType' => 'DataSource', |
7b057b66 EM |
114 | 'dateFormats' => 'DataSource', |
115 | 'savedMapping' => 'DataSource', | |
116 | 'dataSource' => 'DataSource', | |
dfa2f16c EM |
117 | 'dedupe_rule_id' => 'DataSource', |
118 | 'onDuplicate' => 'DataSource', | |
119 | 'disableUSPS' => 'DataSource', | |
9c16701f EM |
120 | 'doGeocodeAddress' => 'DataSource', |
121 | // Note we don't add the save mapping instructions for MapField here | |
122 | // (eg 'updateMapping') - as they really are an action for that form | |
123 | // rather than part of the mapping config. | |
124 | 'mapper' => 'MapField', | |
7b057b66 EM |
125 | ]; |
126 | ||
127 | /** | |
128 | * Get the submitted value, accessing it from whatever form in the flow it is | |
129 | * submitted on. | |
130 | * | |
9d7974eb EM |
131 | * @param string $fieldName |
132 | * | |
133 | * @return mixed|null | |
7b057b66 | 134 | * @throws \CRM_Core_Exception |
9d7974eb EM |
135 | */ |
136 | public function getSubmittedValue(string $fieldName) { | |
7b057b66 EM |
137 | if ($fieldName === 'dataSource') { |
138 | // Hard-coded handling for DataSource as it affects the contents of | |
139 | // getSubmittableFields and can cause a loop. | |
140 | return $this->controller->exportValue('DataSource', 'dataSource'); | |
141 | } | |
142 | $mappedValues = $this->getSubmittableFields(); | |
9d7974eb EM |
143 | if (array_key_exists($fieldName, $mappedValues)) { |
144 | return $this->controller->exportValue($mappedValues[$fieldName], $fieldName); | |
145 | } | |
146 | return parent::getSubmittedValue($fieldName); | |
147 | ||
148 | } | |
149 | ||
7b057b66 EM |
150 | /** |
151 | * Get values submitted on any form in the multi-page import flow. | |
152 | * | |
153 | * @return array | |
154 | */ | |
155 | public function getSubmittedValues(): array { | |
156 | $values = []; | |
157 | foreach (array_keys($this->getSubmittableFields()) as $key) { | |
158 | $values[$key] = $this->getSubmittedValue($key); | |
159 | } | |
160 | return $values; | |
161 | } | |
162 | ||
39dc35d4 EM |
163 | /** |
164 | * Get the available datasource. | |
165 | * | |
166 | * Permission dependent, this will look like | |
167 | * [ | |
168 | * 'CRM_Import_DataSource_CSV' => 'Comma-Separated Values (CSV)', | |
169 | * 'CRM_Import_DataSource_SQL' => 'SQL Query', | |
170 | * ] | |
171 | * | |
172 | * The label is translated. | |
173 | * | |
174 | * @return array | |
175 | */ | |
176 | protected function getDataSources(): array { | |
177 | $dataSources = []; | |
178 | foreach (['CRM_Import_DataSource_SQL', 'CRM_Import_DataSource_CSV'] as $dataSourceClass) { | |
179 | $object = new $dataSourceClass(); | |
180 | if ($object->checkPermission()) { | |
181 | $dataSources[$dataSourceClass] = $object->getInfo()['title']; | |
182 | } | |
183 | } | |
184 | return $dataSources; | |
185 | } | |
186 | ||
d452dfe6 EM |
187 | /** |
188 | * Get the name of the datasource class. | |
189 | * | |
190 | * This function prioritises retrieving from GET and POST over 'submitted'. | |
191 | * The reason for this is the submitted array will hold the previous submissions | |
192 | * data until after buildForm is called. | |
193 | * | |
194 | * This is problematic in the forward->back flow & option changing flow. As in.... | |
195 | * | |
196 | * 1) Load DataSource form - initial default datasource is set to CSV and the | |
197 | * form is via ajax (this calls DataSourceConfig to get the data). | |
198 | * 2) User changes the source to SQL - the ajax updates the html but the | |
199 | * form was built with the expectation that the csv-specific fields would be | |
200 | * required. | |
201 | * 3) When the user submits Quickform calls preProcess and buildForm and THEN | |
202 | * retrieves the submitted values based on what has been added in buildForm. | |
203 | * Only the submitted values for fields added in buildForm are available - but | |
204 | * these have to be added BEFORE the submitted values are determined. Hence | |
205 | * we look in the POST or GET to get the updated value. | |
206 | * | |
207 | * Note that an imminent refactor will involve storing the values in the | |
208 | * civicrm_user_job table - this will hopefully help with a known (not new) | |
209 | * issue whereby the previously submitted values (eg. skipColumnHeader has | |
210 | * been checked or sql has been filled in) are not loaded via the ajax request. | |
211 | * | |
212 | * @return string|null | |
213 | * | |
214 | * @throws \CRM_Core_Exception | |
215 | */ | |
c2562331 | 216 | protected function getDataSourceClassName(): string { |
d452dfe6 EM |
217 | $className = CRM_Utils_Request::retrieveValue( |
218 | 'dataSource', | |
219 | 'String' | |
220 | ); | |
221 | if (!$className) { | |
222 | $className = $this->getSubmittedValue('dataSource'); | |
223 | } | |
224 | if (!$className) { | |
225 | $className = $this->getDefaultDataSource(); | |
226 | } | |
227 | if ($this->getDataSources()[$className]) { | |
228 | return $className; | |
229 | } | |
230 | throw new CRM_Core_Exception('Invalid data source'); | |
231 | } | |
232 | ||
233 | /** | |
234 | * Allow the datasource class to add fields. | |
235 | * | |
236 | * This is called as a snippet in DataSourceConfig and | |
237 | * also from DataSource::buildForm to add the fields such | |
238 | * that quick form picks them up. | |
239 | * | |
240 | * @throws \CRM_Core_Exception | |
241 | */ | |
242 | protected function buildDataSourceFields(): void { | |
7b057b66 EM |
243 | $dataSourceClass = $this->getDataSourceObject(); |
244 | if ($dataSourceClass) { | |
245 | $dataSourceClass->buildQuickForm($this); | |
246 | } | |
247 | } | |
248 | ||
1163561b EM |
249 | /** |
250 | * Flush datasource on re-submission of the form. | |
251 | * | |
252 | * If the form has been re-submitted the datasource might have changed. | |
253 | * We tell the dataSource class to remove any tables (and potentially files) | |
254 | * created last form submission. | |
255 | * | |
256 | * If the DataSource in use is unchanged (ie still CSV or still SQL) | |
257 | * we also pass in the new variables. In theory it could decide that they | |
258 | * have not actually changed and it doesn't need to do any cleanup. | |
259 | * | |
260 | * In practice the datasource classes blast away as they always have for now | |
261 | * - however, the sql class, for example, might realise the fields it cares | |
262 | * about are unchanged and not flush the table. | |
263 | * | |
264 | * @throws \API_Exception | |
265 | * @throws \CRM_Core_Exception | |
266 | */ | |
267 | protected function flushDataSource(): void { | |
268 | // If the form has been resubmitted the datasource might have changed. | |
269 | // We give the datasource a chance to clean up any tables it might have | |
270 | // created. If we are still using the same type of datasource (e.g still | |
271 | // an sql query | |
272 | $oldDataSource = $this->getUserJobSubmittedValues()['dataSource']; | |
273 | $oldDataSourceObject = new $oldDataSource($this->getUserJobID()); | |
274 | $newParams = $this->getSubmittedValue('dataSource') === $oldDataSource ? $this->getSubmittedValues() : []; | |
275 | $oldDataSourceObject->purge($newParams); | |
276 | } | |
277 | ||
7b057b66 EM |
278 | /** |
279 | * Get the relevant datasource object. | |
280 | * | |
281 | * @return \CRM_Import_DataSource|null | |
282 | * | |
283 | * @throws \CRM_Core_Exception | |
284 | */ | |
285 | protected function getDataSourceObject(): ?CRM_Import_DataSource { | |
286 | $className = $this->getDataSourceClassName(); | |
287 | if ($className) { | |
288 | /* @var CRM_Import_DataSource $dataSource */ | |
289 | return new $className($this->getUserJobID()); | |
290 | } | |
291 | return NULL; | |
292 | } | |
293 | ||
294 | /** | |
295 | * Allow the datasource class to add fields. | |
296 | * | |
297 | * This is called as a snippet in DataSourceConfig and | |
298 | * also from DataSource::buildForm to add the fields such | |
299 | * that quick form picks them up. | |
300 | * | |
301 | * @throws \CRM_Core_Exception | |
302 | */ | |
303 | protected function getDataSourceFields(): array { | |
d452dfe6 EM |
304 | $className = $this->getDataSourceClassName(); |
305 | if ($className) { | |
7b057b66 | 306 | /* @var CRM_Import_DataSource $dataSourceClass */ |
d452dfe6 | 307 | $dataSourceClass = new $className(); |
7b057b66 | 308 | return $dataSourceClass->getSubmittableFields(); |
d452dfe6 | 309 | } |
7b057b66 | 310 | return []; |
d452dfe6 EM |
311 | } |
312 | ||
313 | /** | |
314 | * Get the default datasource. | |
315 | * | |
316 | * @return string | |
317 | */ | |
318 | protected function getDefaultDataSource(): string { | |
319 | return 'CRM_Import_DataSource_CSV'; | |
320 | } | |
321 | ||
7b057b66 EM |
322 | /** |
323 | * Get the fields that can be submitted in the Import form flow. | |
324 | * | |
325 | * These could be on any form in the flow & are accessed the same way from | |
326 | * all forms. | |
327 | * | |
328 | * @return string[] | |
329 | * @throws \CRM_Core_Exception | |
330 | */ | |
331 | protected function getSubmittableFields(): array { | |
332 | $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource'); | |
333 | return array_merge($this->submittableFields, $dataSourceFields); | |
334 | } | |
335 | ||
6e78138c EM |
336 | /** |
337 | * Get the contact type selected for the import (on the datasource form). | |
338 | * | |
339 | * @return string | |
340 | * e.g Individual, Organization, Household. | |
341 | * | |
342 | * @throws \CRM_Core_Exception | |
343 | */ | |
344 | protected function getContactType(): string { | |
345 | $contactTypeMapping = [ | |
346 | CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual', | |
347 | CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household', | |
348 | CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization', | |
349 | ]; | |
350 | return $contactTypeMapping[$this->getSubmittedValue('contactType')]; | |
351 | } | |
352 | ||
80cb71bb EM |
353 | /** |
354 | * Get the contact sub type selected for the import (on the datasource form). | |
355 | * | |
356 | * @return string|null | |
357 | * e.g Staff. | |
358 | * | |
359 | * @throws \CRM_Core_Exception | |
360 | */ | |
361 | protected function getContactSubType(): ?string { | |
362 | return $this->getSubmittedValue('contactSubType'); | |
363 | } | |
364 | ||
7b057b66 EM |
365 | /** |
366 | * Create a user job to track the import. | |
367 | * | |
368 | * @return int | |
369 | * | |
370 | * @throws \API_Exception | |
371 | */ | |
372 | protected function createUserJob(): int { | |
373 | $id = UserJob::create(FALSE) | |
374 | ->setValues([ | |
375 | 'created_id' => CRM_Core_Session::getLoggedInContactID(), | |
376 | 'type_id:name' => 'contact_import', | |
377 | 'status_id:name' => 'draft', | |
378 | // This suggests the data could be cleaned up after this. | |
379 | 'expires_date' => '+ 1 week', | |
380 | 'metadata' => [ | |
381 | 'submitted_values' => $this->getSubmittedValues(), | |
382 | ], | |
383 | ]) | |
384 | ->execute() | |
385 | ->first()['id']; | |
386 | $this->setUserJobID($id); | |
387 | return $id; | |
388 | } | |
389 | ||
390 | /** | |
391 | * @param string $key | |
392 | * @param array $data | |
393 | * | |
394 | * @throws \API_Exception | |
395 | * @throws \Civi\API\Exception\UnauthorizedException | |
396 | */ | |
397 | protected function updateUserJobMetadata(string $key, array $data): void { | |
398 | $metaData = array_merge( | |
399 | $this->getUserJob()['metadata'], | |
400 | [$key => $data] | |
401 | ); | |
402 | UserJob::update(FALSE) | |
403 | ->addWhere('id', '=', $this->getUserJobID()) | |
404 | ->setValues(['metadata' => $metaData]) | |
405 | ->execute(); | |
406 | $this->userJob['metadata'] = $metaData; | |
407 | } | |
408 | ||
4a01628c EM |
409 | /** |
410 | * Get column headers for the datasource or empty array if none apply. | |
411 | * | |
412 | * This would be the first row of a csv or the fields in an sql query. | |
413 | * | |
414 | * If the csv does not have a header row it will be empty. | |
415 | * | |
416 | * @return array | |
417 | * | |
418 | * @throws \API_Exception | |
419 | * @throws \CRM_Core_Exception | |
420 | */ | |
421 | protected function getColumnHeaders(): array { | |
422 | return $this->getDataSourceObject()->getColumnHeaders(); | |
423 | } | |
424 | ||
425 | /** | |
426 | * Get the number of importable columns in the data source. | |
427 | * | |
428 | * @return int | |
429 | * | |
430 | * @throws \API_Exception | |
431 | * @throws \CRM_Core_Exception | |
432 | */ | |
433 | protected function getNumberOfColumns(): int { | |
434 | return $this->getDataSourceObject()->getNumberOfColumns(); | |
435 | } | |
436 | ||
437 | /** | |
438 | * Get x data rows from the datasource. | |
439 | * | |
440 | * At this stage we are fetching from what has been stored in the form | |
441 | * during `postProcess` on the DataSource form. | |
442 | * | |
443 | * In the future we will use the dataSource object, likely | |
444 | * supporting offset as well. | |
445 | * | |
99e3c5f7 EM |
446 | * @return array|int |
447 | * One or more of the statues available - e.g | |
448 | * CRM_Import_Parser::VALID | |
da8d3d49 | 449 | * or [CRM_Import_Parser::ERROR, CRM_Import_Parser::VALID] |
4a01628c | 450 | * |
99e3c5f7 EM |
451 | * @throws \CRM_Core_Exception |
452 | * @throws \API_Exception | |
453 | */ | |
454 | protected function getDataRows($statuses = [], int $limit = 0): array { | |
455 | $statuses = (array) $statuses; | |
456 | return $this->getDataSourceObject()->setLimit($limit)->setStatuses($statuses)->getRows(); | |
457 | } | |
458 | ||
459 | /** | |
460 | * Get the number of rows with the specified status. | |
4a01628c | 461 | * |
99e3c5f7 EM |
462 | * @param array|int $statuses |
463 | * | |
464 | * @return int | |
465 | * | |
466 | * @throws \API_Exception | |
4a01628c | 467 | * @throws \CRM_Core_Exception |
99e3c5f7 EM |
468 | */ |
469 | protected function getRowCount($statuses = []) { | |
470 | $statuses = (array) $statuses; | |
471 | return $this->getDataSourceObject()->getRowCount($statuses); | |
472 | } | |
473 | ||
474 | /** | |
475 | * Outputs and downloads the csv of outcomes from an import job. | |
476 | * | |
477 | * This gets the rows from the temp table that match the relevant status | |
478 | * and output them as a csv. | |
479 | * | |
4a01628c | 480 | * @throws \API_Exception |
99e3c5f7 EM |
481 | * @throws \League\Csv\CannotInsertRecord |
482 | * @throws \CRM_Core_Exception | |
483 | */ | |
484 | public static function outputCSV(): void { | |
485 | $userJobID = CRM_Utils_Request::retrieveValue('user_job_id', 'Integer', NULL, TRUE); | |
486 | $status = CRM_Utils_Request::retrieveValue('status', 'String', NULL, TRUE); | |
487 | $saveFileName = CRM_Import_Parser::saveFileName($status); | |
488 | ||
489 | $form = new CRM_Import_Forms(); | |
490 | $form->controller = new CRM_Core_Controller(); | |
491 | $form->set('user_job_id', $userJobID); | |
492 | ||
493 | $form->getUserJob(); | |
494 | $writer = Writer::createFromFileObject(new SplTempFileObject()); | |
495 | $headers = $form->getColumnHeaders(); | |
496 | if ($headers) { | |
497 | array_unshift($headers, ts('Reason')); | |
498 | array_unshift($headers, ts('Line Number')); | |
499 | $writer->insertOne($headers); | |
500 | } | |
501 | $writer->addFormatter(['CRM_Import_Forms', 'reorderOutput']); | |
502 | // Note this might be more inefficient that iterating the result | |
503 | // set & doing insertOne - possibly something to explore later. | |
504 | $writer->insertAll($form->getDataRows($status)); | |
505 | ||
506 | CRM_Utils_System::setHttpHeader('Cache-Control', 'must-revalidate, post-check=0, pre-check=0'); | |
507 | CRM_Utils_System::setHttpHeader('Content-Description', 'File Transfer'); | |
508 | CRM_Utils_System::setHttpHeader('Content-Type', 'text/csv; charset=UTF-8'); | |
509 | $writer->output($saveFileName); | |
510 | CRM_Utils_System::civiExit(); | |
511 | } | |
512 | ||
513 | /** | |
514 | * When outputting the row as a csv, more the last 2 rows to the start. | |
515 | * | |
516 | * This is because the id and status message fields are at the end. It may make sense | |
517 | * to move them to the start later, when order code cleanup has happened... | |
518 | * | |
519 | * @param array $record | |
520 | */ | |
521 | public static function reorderOutput(array $record): array { | |
522 | $rowNumber = array_pop($record); | |
523 | $message = array_pop($record); | |
524 | // Also pop off the status - but we are not going to use this at this stage. | |
525 | array_pop($record); | |
5e21b588 EM |
526 | // Related entities |
527 | array_pop($record); | |
528 | // Entity_id | |
529 | array_pop($record); | |
99e3c5f7 EM |
530 | array_unshift($record, $message); |
531 | array_unshift($record, $rowNumber); | |
532 | return $record; | |
533 | } | |
534 | ||
535 | /** | |
536 | * Get the url to download the relevant csv file. | |
537 | * @param string $status | |
538 | * | |
539 | * @return string | |
4a01628c | 540 | */ |
99e3c5f7 EM |
541 | protected function getDownloadURL(string $status): string { |
542 | return CRM_Utils_System::url('civicrm/import/outcome', [ | |
543 | 'user_job_id' => $this->get('user_job_id'), | |
544 | 'status' => $status, | |
545 | 'reset' => 1, | |
546 | ]); | |
4a01628c EM |
547 | } |
548 | ||
52bd01f5 EM |
549 | /** |
550 | * Get the fields available for import selection. | |
551 | * | |
552 | * @return array | |
553 | * e.g ['first_name' => 'First Name', 'last_name' => 'Last Name'.... | |
554 | * | |
555 | * @throws \API_Exception | |
556 | */ | |
557 | protected function getAvailableFields(): array { | |
73edfc10 EM |
558 | return $this->getParser()->getAvailableFields(); |
559 | } | |
560 | ||
561 | /** | |
562 | * Get an instance of the parser class. | |
563 | * | |
564 | * @return \CRM_Contact_Import_Parser_Contact|\CRM_Contribute_Import_Parser_Contribution | |
565 | */ | |
566 | protected function getParser() { | |
567 | return NULL; | |
52bd01f5 EM |
568 | } |
569 | ||
2a879010 EM |
570 | /** |
571 | * Get the mapped fields as an array of labels. | |
572 | * | |
573 | * e.g | |
574 | * ['First Name', 'Employee Of - First Name', 'Home - Street Address'] | |
575 | * | |
576 | * @return array | |
577 | * @throws \API_Exception | |
578 | * @throws \CRM_Core_Exception | |
579 | */ | |
580 | protected function getMappedFieldLabels(): array { | |
581 | $mapper = []; | |
d00da884 | 582 | $parser = $this->getParser(); |
2a879010 EM |
583 | foreach ($this->getSubmittedValue('mapper') as $columnNumber => $mappedField) { |
584 | $mapper[$columnNumber] = $parser->getMappedFieldLabel($parser->getMappingFieldFromMapperInput($mappedField, 0, $columnNumber)); | |
585 | } | |
586 | return $mapper; | |
587 | } | |
588 | ||
9d7974eb | 589 | } |