Chris@0: <?php
Chris@0: 
Chris@0: /**
Chris@0:  * @file
Chris@0:  * Command-line tools to aid performing and developing migrations.
Chris@0:  */
Chris@0: 
Chris@0: use Drupal\Component\Plugin\Exception\PluginException;
Chris@0: use Drupal\Component\Plugin\Exception\PluginNotFoundException;
Chris@0: use Drupal\Component\Utility\Unicode;
Chris@0: use Drupal\Core\Database\ConnectionNotDefinedException;
Chris@0: use Drupal\migrate\Exception\RequirementsException;
Chris@0: use Drupal\migrate\Plugin\MigrationInterface;
Chris@0: use Drupal\migrate_tools\MigrateExecutable;
Chris@0: use Drupal\migrate_tools\DrushLogMigrateMessage;
Chris@0: use Drupal\Core\Datetime\DateFormatter;
Chris@0: 
Chris@0: /**
Chris@0:  * Implements hook_drush_command().
Chris@0:  */
Chris@0: function migrate_tools_drush_command() {
Chris@0:   $items['migrate-status'] = [
Chris@0:     'description' => 'List all migrations with current status.',
Chris@0:     'options' => [
Chris@0:       'group' => 'Name of the migration group to list',
Chris@0:       'names-only' => 'Only return names, not all the details (faster)',
Chris@0:     ],
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Restrict to a comma-separated list of migrations. Optional',
Chris@0:     ],
Chris@0:     'examples' => [
Chris@0:       'migrate-status' => 'Retrieve status for all migrations',
Chris@0:       'migrate-status --group=beer' => 'Retrieve status for all migrations in a given group',
Chris@0:       'migrate-status BeerTerm,BeerNode' => 'Retrieve status for specific migrations',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['ms'],
Chris@0:   ];
Chris@0: 
Chris@0:   $items['migrate-import'] = [
Chris@0:     'description' => 'Perform one or more migration processes.',
Chris@0:     'options' => [
Chris@0:       'all' => 'Process all migrations.',
Chris@0:       'group' => 'Name of the migration group to import',
Chris@0:       'limit' => 'Limit on the number of items to process in each migration',
Chris@0:       'feedback' => 'Frequency of progress messages, in items processed',
Chris@0:       'idlist' => 'Comma-separated list of IDs to import',
Chris@0:       'update' => ' In addition to processing unprocessed items from the source, update previously-imported items with the current data',
Chris@0:       'force' => 'Force an operation to run, even if all dependencies are not satisfied',
Chris@0:       'execute-dependencies' => 'Execute all dependent migrations first.',
Chris@0:     ],
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Name of migration(s) to import. Delimit multiple using commas.',
Chris@0:     ],
Chris@0:     'examples' => [
Chris@0:       'migrate-import --all' => 'Perform all migrations',
Chris@0:       'migrate-import --group=beer' => 'Import all migrations in the beer group',
Chris@0:       'migrate-import BeerTerm,BeerNode' => 'Import new terms and nodes',
Chris@0:       'migrate-import BeerUser --limit=2' => 'Import no more than 2 users',
Chris@0:       'migrate-import BeerUser --idlist=5' => 'Import the user record with source ID 5',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['mi'],
Chris@0:   ];
Chris@0: 
Chris@0:   $items['migrate-rollback'] = array(
Chris@0:     'description' => 'Rollback one or more migrations.',
Chris@0:     'options' => array(
Chris@0:       'all' => 'Process all migrations.',
Chris@0:       'group' => 'Name of the migration group to rollback',
Chris@0:       'feedback' => 'Frequency of progress messages, in items processed',
Chris@0:     ),
Chris@0:     'arguments' => array(
Chris@0:       'migration' => 'Name of migration(s) to rollback. Delimit multiple using commas.',
Chris@0:     ),
Chris@0:     'examples' => array(
Chris@0:       'migrate-rollback --all' => 'Perform all migrations',
Chris@0:       'migrate-rollback --group=beer' => 'Rollback all migrations in the beer group',
Chris@0:       'migrate-rollback BeerTerm,BeerNode' => 'Rollback imported terms and nodes',
Chris@0:     ),
Chris@0:     'drupal dependencies' => array('migrate_tools'),
Chris@0:     'aliases' => array('mr'),
Chris@0:   );
Chris@0: 
Chris@0:   $items['migrate-stop'] = [
Chris@0:     'description' => 'Stop an active migration operation.',
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Name of migration to stop',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['mst'],
Chris@0:   ];
Chris@0: 
Chris@0:   $items['migrate-reset-status'] = [
Chris@0:     'description' => 'Reset a active migration\'s status to idle.',
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Name of migration to reset',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['mrs'],
Chris@0:   ];
Chris@0: 
Chris@0:   $items['migrate-messages'] = [
Chris@0:     'description' => 'View any messages associated with a migration.',
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Name of the migration',
Chris@0:     ],
Chris@0:     'options' => [
Chris@0:       'csv' => 'Export messages as a CSV'
Chris@0:     ],
Chris@0:     'examples' => [
Chris@0:       'migrate-messages MyNode' => 'Show all messages for the MyNode migration',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['mmsg'],
Chris@0:   ];
Chris@0: 
Chris@0:   $items['migrate-fields-source'] = [
Chris@0:     'description' => 'List the fields available for mapping in a source.',
Chris@0:     'arguments' => [
Chris@0:       'migration' => 'Name of the migration',
Chris@0:     ],
Chris@0:     'examples' => [
Chris@0:       'migrate-fields-source my_node' => 'List fields for the source in the my_node migration',
Chris@0:     ],
Chris@0:     'drupal dependencies' => ['migrate_tools'],
Chris@0:     'aliases' => ['mfs'],
Chris@0:   ];
Chris@0: 
Chris@0:   return $items;
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_names
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_status($migration_names = '') {
Chris@0:   $group_name = drush_get_option('group');
Chris@0:   $names_only = drush_get_option('names-only');
Chris@0: 
Chris@0:   $migrations = drush_migrate_tools_migration_list($group_name, $migration_names);
Chris@0: 
Chris@0:   $table = [];
Chris@0:   // Take it one group at a time, listing the migrations within each group.
Chris@0:   foreach ($migrations as $group_id => $migration_list) {
Chris@0:     if ($names_only) {
Chris@0:       $table[] = [
Chris@0:         dt('Group: @name', array('@name' => $group_id))
Chris@0:       ];
Chris@0:     }
Chris@0:     else {
Chris@0:       $table[] = [
Chris@0:         dt('Group: @name', array('@name' => $group_id)),
Chris@0:         dt('Status'),
Chris@0:         dt('Total'),
Chris@0:         dt('Imported'),
Chris@0:         dt('Unprocessed'),
Chris@0:         dt('Last imported'),
Chris@0:       ];
Chris@0:     }
Chris@0:     foreach ($migration_list as $migration_id => $migration) {
Chris@0:       try {
Chris@0:         $map = $migration->getIdMap();
Chris@0:         $imported = $map->importedCount();
Chris@0:         $source_plugin = $migration->getSourcePlugin();
Chris@0:       }
Chris@0:       catch (Exception $e) {
Chris@0:         drush_log(dt('Failure retrieving information on @migration: @message',
Chris@0:           ['@migration' => $migration_id, '@message' => $e->getMessage()]));
Chris@0:         continue;
Chris@0:       }
Chris@0:       try {
Chris@0:         $source_rows = $source_plugin->count();
Chris@0:         // -1 indicates uncountable sources.
Chris@0:         if ($source_rows == -1) {
Chris@0:           $source_rows = dt('N/A');
Chris@0:           $unprocessed = dt('N/A');
Chris@0:         }
Chris@0:         else {
Chris@0:           $unprocessed = $source_rows - $map->processedCount();
Chris@0:         }
Chris@0:       }
Chris@0:       catch (Exception $e) {
Chris@0:         drush_print($e->getMessage());
Chris@0:         drush_log(dt('Could not retrieve source count from @migration: @message',
Chris@0:           ['@migration' => $migration_id, '@message' => $e->getMessage()]));
Chris@0:         $source_rows = dt('N/A');
Chris@0:         $unprocessed = dt('N/A');
Chris@0:       }
Chris@0: 
Chris@0:       if ($names_only) {
Chris@0:         $table[] = [$migration_id];
Chris@0:       }
Chris@0:       else {
Chris@0:         $status = $migration->getStatusLabel();
Chris@0:         $migrate_last_imported_store = \Drupal::keyValue('migrate_last_imported');
Chris@0:         $last_imported =  $migrate_last_imported_store->get($migration->id(), FALSE);
Chris@0:         if ($last_imported) {
Chris@0:           /** @var DateFormatter $date_formatter */
Chris@0:           $date_formatter = \Drupal::service('date.formatter');
Chris@0:           $last_imported = $date_formatter->format($last_imported / 1000,
Chris@0:             'custom', 'Y-m-d H:i:s');
Chris@0:         }
Chris@0:         else {
Chris@0:           $last_imported = '';
Chris@0:         }
Chris@0:         $table[] = [$migration_id, $status, $source_rows, $imported, $unprocessed, $last_imported];
Chris@0:       }
Chris@0:     }
Chris@0:   }
Chris@0:   drush_print_table($table);
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_names
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_import($migration_names = '') {
Chris@0:   $group_name = drush_get_option('group');
Chris@0:   $all = drush_get_option('all');
Chris@0:   $options = [];
Chris@0:   if (!$all && !$group_name && !$migration_names) {
Chris@0:     drush_set_error('MIGRATE_ERROR', dt('You must specify --all, --group, or one or more migration names separated by commas'));
Chris@0:     return;
Chris@0:   }
Chris@0: 
Chris@0:   foreach (['limit', 'feedback', 'idlist', 'update', 'force'] as $option) {
Chris@0:     if (drush_get_option($option)) {
Chris@0:       $options[$option] = drush_get_option($option);
Chris@0:     }
Chris@0:   }
Chris@0: 
Chris@0:   $migrations = drush_migrate_tools_migration_list($group_name, $migration_names);
Chris@0: 
Chris@0:   // Take it one group at a time, importing the migrations within each group.
Chris@0:   foreach ($migrations as $group_id => $migration_list) {
Chris@0:     array_walk($migration_list, '_drush_migrate_tools_execute_migration', $options);
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Executes a single migration. If the --execute-dependencies option was given,
Chris@0:  * the migration's dependencies will also be executed first.
Chris@0:  *
Chris@0:  * @param \Drupal\migrate\Plugin\MigrationInterface $migration
Chris@0:  *  The migration to execute.
Chris@0:  * @param string $migration_id
Chris@0:  *  The migration ID (not used, just an artifact of array_walk()).
Chris@0:  * @param array $options
Chris@0:  *  Additional options for the migration.
Chris@0:  */
Chris@0: function _drush_migrate_tools_execute_migration(MigrationInterface $migration, $migration_id,  array $options = []) {
Chris@0:   $log = new DrushLogMigrateMessage();
Chris@0: 
Chris@0:   if (drush_get_option('execute-dependencies')) {
Chris@0:     if ($required_IDS = $migration->get('requirements')) {
Chris@0:       $manager = \Drupal::service('plugin.manager.config_entity_migration');
Chris@0:       $required_migrations = $manager->createInstances($required_IDS);
Chris@0:       $dependency_options = array_merge($options, ['is_dependency' => TRUE]);
Chris@0:       array_walk($required_migrations, __FUNCTION__, $dependency_options);
Chris@0:     }
Chris@0:   }
Chris@0:   if ($options['force']) {
Chris@0:     $migration->set('requirements', []);
Chris@0:   }
Chris@0:   if ($options['update']) {
Chris@0:     $migration->getIdMap()->prepareUpdate();
Chris@0:   }
Chris@0:   $executable = new MigrateExecutable($migration, $log, $options);
Chris@0:   // drush_op() provides --simulate support
Chris@0:   drush_op(array($executable, 'import'));
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_names
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_rollback($migration_names = '') {
Chris@0:   $group_name = drush_get_option('group');
Chris@0:   $all = drush_get_option('all');
Chris@0:   $options = [];
Chris@0:   if (!$all && !$group_name && !$migration_names) {
Chris@0:     drush_set_error('MIGRATE_ERROR', dt('You must specify --all, --group, or one or more migration names separated by commas'));
Chris@0:     return;
Chris@0:   }
Chris@0: 
Chris@0:   if (drush_get_option('feedback')) {
Chris@0:     $options['feedback'] = drush_get_option('feedback');
Chris@0:   }
Chris@0: 
Chris@0:   $log = new DrushLogMigrateMessage();
Chris@0: 
Chris@0:   $migrations = drush_migrate_tools_migration_list($group_name, $migration_names);
Chris@0: 
Chris@0:   // Take it one group at a time, rolling back the migrations within each group.
Chris@0:   foreach ($migrations as $group_id => $migration_list) {
Chris@0:     // Roll back in reverse order.
Chris@0:     $migration_list = array_reverse($migration_list);
Chris@0:     foreach ($migration_list as $migration_id => $migration) {
Chris@0:       $executable = new MigrateExecutable($migration, $log, $options);
Chris@0:       // drush_op() provides --simulate support.
Chris@0:       drush_op(array($executable, 'rollback'));
Chris@0:     }
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_id
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_stop($migration_id = '') {
Chris@0:   /** @var MigrationInterface $migration */
Chris@0:   $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
Chris@0:   if ($migration) {
Chris@0:     $status = $migration->getStatus();
Chris@0:     switch ($status) {
Chris@0:       case MigrationInterface::STATUS_IDLE:
Chris@0:         drush_log(dt('Migration @id is idle', ['@id' => $migration_id]), 'warning');
Chris@0:         break;
Chris@0:       case MigrationInterface::STATUS_DISABLED:
Chris@0:         drush_log(dt('Migration @id is disabled', ['@id' => $migration_id]), 'warning');
Chris@0:         break;
Chris@0:       case MigrationInterface::STATUS_STOPPING:
Chris@0:         drush_log(dt('Migration @id is already stopping', ['@id' => $migration_id]), 'warning');
Chris@0:         break;
Chris@0:       default:
Chris@0:         $migration->interruptMigration(MigrationInterface::RESULT_STOPPED);
Chris@0:         drush_log(dt('Migration @id requested to stop', ['@id' => $migration_id]), 'success');
Chris@0:         break;
Chris@0:     }
Chris@0:   }
Chris@0:   else {
Chris@0:     drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_id
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_reset_status($migration_id = '') {
Chris@0:   /** @var MigrationInterface $migration */
Chris@0:   $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
Chris@0:   if ($migration) {
Chris@0:     $status = $migration->getStatus();
Chris@0:     if ($status == MigrationInterface::STATUS_IDLE) {
Chris@0:       drush_log(dt('Migration @id is already Idle', ['@id' => $migration_id]), 'warning');
Chris@0:     }
Chris@0:     else {
Chris@0:       $migration->setStatus(MigrationInterface::STATUS_IDLE);
Chris@0:       drush_log(dt('Migration @id reset to Idle', ['@id' => $migration_id]), 'status');
Chris@0:     }
Chris@0:   }
Chris@0:   else {
Chris@0:     drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_id
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_messages($migration_id) {
Chris@0:   /** @var MigrationInterface $migration */
Chris@0:   $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
Chris@0:   if ($migration) {
Chris@0:     $map = $migration->getIdMap();
Chris@0:     $first = TRUE;
Chris@0:     $table = [];
Chris@0:     foreach ($map->getMessageIterator() as $row) {
Chris@0:       unset($row->msgid);
Chris@0:       if ($first) {
Chris@0:         // @todo: Ideally, replace sourceid* with source key names. Or, should
Chris@0:         // getMessageIterator() do that?
Chris@0:         foreach ($row as $column => $value) {
Chris@0:           $table[0][] = $column;
Chris@0:         }
Chris@0:         $first = FALSE;
Chris@0:       }
Chris@0:       $table[] = (array)$row;
Chris@0:     }
Chris@0:     if (empty($table)) {
Chris@0:       drush_log(dt('No messages for this migration'), 'status');
Chris@0:     }
Chris@0:     else {
Chris@0:       if (drush_get_option('csv')) {
Chris@0:         foreach ($table as $row) {
Chris@0:           fputcsv(STDOUT, $row);
Chris@0:         }
Chris@0:       }
Chris@0:       else {
Chris@0:         $widths = [];
Chris@0:         foreach ($table[0] as $header) {
Chris@0:           $widths[] = strlen($header) + 1;
Chris@0:         }
Chris@0:         drush_print_table($table, TRUE, $widths);
Chris@0:       }
Chris@0:     }
Chris@0:   }
Chris@0:   else {
Chris@0:     drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * @param string $migration_id
Chris@0:  */
Chris@0: function drush_migrate_tools_migrate_fields_source($migration_id) {
Chris@0:   /** @var MigrationInterface $migration */
Chris@0:   $migration = \Drupal::service('plugin.manager.migration')->createInstance($migration_id);
Chris@0:   if ($migration) {
Chris@0:     $source = $migration->getSourcePlugin();
Chris@0:     $table = [];
Chris@0:     foreach ($source->fields() as $machine_name => $description) {
Chris@0:       $table[] = [strip_tags($description), $machine_name];
Chris@0:     }
Chris@0:     drush_print_table($table);
Chris@0:   }
Chris@0:   else {
Chris@0:     drush_log(dt('Migration @id does not exist', ['@id' => $migration_id]), 'error');
Chris@0:   }
Chris@0: }
Chris@0: 
Chris@0: /**
Chris@0:  * Retrieve a list of active migrations.
Chris@0:  *
Chris@0:  * @param string $group_id
Chris@0:  *  Group machine name - if present, return only migrations in this group.
Chris@0:  * @param string $migration_ids
Chris@0:  *  Comma-separated list of migrations - if present, return only these migrations.
Chris@0:  *
Chris@0:  * @return MigrationInterface[][]
Chris@0:  *   An array keyed by migration group, each value containing an array of migrations.
Chris@0:  */
Chris@0: function drush_migrate_tools_migration_list($group_id = '', $migration_ids = '') {
Chris@0:   if (!empty($migration_ids)) {
Chris@0:     $migration_ids = explode(',', Unicode::strtolower($migration_ids));
Chris@0:   }
Chris@0:   else {
Chris@0:     $migration_ids = [];
Chris@0:   }
Chris@0: 
Chris@0:   $manager = \Drupal::service('plugin.manager.config_entity_migration');
Chris@0:   $plugins = $manager->createInstances([]);
Chris@0:   $migrations = [];
Chris@0:   foreach ($plugins as $id => $migration) {
Chris@0:     $configured_group_id = $migration->get('migration_group');
Chris@0:     if (empty($configured_group_id)) {
Chris@0:       $configured_group_id = 'default';
Chris@0:     }
Chris@0:     if (empty($group_id) || $group_id == $configured_group_id) {
Chris@0:       if (empty($migration_ids) || in_array(Unicode::strtolower($id), $migration_ids)) {
Chris@0:         $migrations[$configured_group_id][$id] = $migration;
Chris@0:       }
Chris@0:     }
Chris@0:   }
Chris@0:   return $migrations;
Chris@0: }