annotate modules/contrib/migrate_plus/src/Plugin/migrate/source/Url.php @ 7:848c88cfe644

More layout
author Chris Cannam
date Fri, 05 Jan 2018 13:59:44 +0000
parents 4c8ae668cc8c
children
rev   line source
Chris@0 1 <?php
Chris@0 2
Chris@0 3 namespace Drupal\migrate_plus\Plugin\migrate\source;
Chris@0 4
Chris@0 5 use Drupal\migrate\Plugin\MigrationInterface;
Chris@0 6 use Drupal\migrate_plus\DataParserPluginInterface;
Chris@0 7
Chris@0 8 /**
Chris@0 9 * Source plugin for retrieving data via URLs.
Chris@0 10 *
Chris@0 11 * @MigrateSource(
Chris@0 12 * id = "url"
Chris@0 13 * )
Chris@0 14 */
Chris@0 15 class Url extends SourcePluginExtension {
Chris@0 16
Chris@0 17 /**
Chris@0 18 * The source URLs to retrieve.
Chris@0 19 *
Chris@0 20 * @var array
Chris@0 21 */
Chris@0 22 protected $sourceUrls = [];
Chris@0 23
Chris@0 24 /**
Chris@0 25 * The data parser plugin.
Chris@0 26 *
Chris@0 27 * @var \Drupal\migrate_plus\DataParserPluginInterface
Chris@0 28 */
Chris@0 29 protected $dataParserPlugin;
Chris@0 30
Chris@0 31 /**
Chris@0 32 * {@inheritdoc}
Chris@0 33 */
Chris@0 34 public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
Chris@0 35 if (!is_array($configuration['urls'])) {
Chris@0 36 $configuration['urls'] = [$configuration['urls']];
Chris@0 37 }
Chris@0 38 parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
Chris@0 39
Chris@0 40 $this->sourceUrls = $configuration['urls'];
Chris@0 41
Chris@0 42 // Set a default Accept header.
Chris@0 43 /* $this->headers = array_merge(['Accept' => 'application/json'],
Chris@0 44 $configuration['headers'] ?: []);*/
Chris@0 45
Chris@0 46 // See if this is a paged response with next links. If so, add to the source_urls array.
Chris@0 47 /* foreach ( (array) $configuration['urls'] as $url) {
Chris@0 48 $this->sourceUrls += $this->getNextLinks($url);
Chris@0 49 }*/
Chris@0 50 }
Chris@0 51
Chris@0 52 /**
Chris@0 53 * Return a string representing the source URLs.
Chris@0 54 *
Chris@0 55 * @return string
Chris@0 56 * Comma-separated list of URLs being imported.
Chris@0 57 */
Chris@0 58 public function __toString() {
Chris@0 59 // This could cause a problem when using a lot of urls, may need to hash.
Chris@0 60 $urls = implode(', ', $this->sourceUrls);
Chris@0 61 return $urls;
Chris@0 62 }
Chris@0 63
Chris@0 64 /**
Chris@0 65 * Returns the initialized data parser plugin.
Chris@0 66 *
Chris@0 67 * @return \Drupal\migrate_plus\DataParserPluginInterface
Chris@0 68 * The data parser plugin.
Chris@0 69 */
Chris@0 70 public function getDataParserPlugin() {
Chris@0 71 if (!isset($this->dataParserPlugin)) {
Chris@0 72 $this->dataParserPlugin = \Drupal::service('plugin.manager.migrate_plus.data_parser')->createInstance($this->configuration['data_parser_plugin'], $this->configuration);
Chris@0 73 }
Chris@0 74 return $this->dataParserPlugin;
Chris@0 75 }
Chris@0 76
Chris@0 77 /**
Chris@0 78 * Creates and returns a filtered Iterator over the documents.
Chris@0 79 *
Chris@0 80 * @return \Iterator
Chris@0 81 * An iterator over the documents providing source rows that match the
Chris@0 82 * configured item_selector.
Chris@0 83 */
Chris@0 84 protected function initializeIterator() {
Chris@0 85 return $this->getDataParserPlugin();
Chris@0 86 }
Chris@0 87
Chris@0 88 /**
Chris@0 89 * Collect an array of next links from a paged response.
Chris@0 90 */
Chris@0 91 /* protected function getNextLinks($url) {
Chris@0 92 $urls = array();
Chris@0 93 $more = TRUE;
Chris@0 94 while ($more == TRUE) {
Chris@0 95 $response = $this->dataParserPlugin->getDataFetcher()->getResponse($url);
Chris@0 96 if ($url = $this->getNextFromHeaders($response)) {
Chris@0 97 $urls[] = $url;
Chris@0 98 }
Chris@0 99 elseif ($url = $this->getNextFromLinks($response)) {
Chris@0 100 $urls[] = $url;
Chris@0 101 }
Chris@0 102 else {
Chris@0 103 $more = FALSE;
Chris@0 104 }
Chris@0 105 }
Chris@0 106 return $urls;
Chris@0 107 }
Chris@0 108 */
Chris@0 109 /**
Chris@0 110 * See if the next link is in a 'links' group in the response.
Chris@0 111 *
Chris@0 112 * @param \Psr\Http\Message\ResponseInterface $response
Chris@0 113 */
Chris@0 114 /* protected function getNextFromLinks(ResponseInterface $response) {
Chris@0 115 $body = json_decode($response->getBody(), TRUE);
Chris@0 116 if (!empty($body['links']) && array_key_exists('next', $body['links'])) {
Chris@0 117 return $body['links']['next'];
Chris@0 118 }
Chris@0 119 return FALSE;
Chris@0 120 }
Chris@0 121 */
Chris@0 122 /**
Chris@0 123 * See if the next link is in the header.
Chris@0 124 *
Chris@0 125 * @param \Psr\Http\Message\ResponseInterface $response
Chris@0 126 */
Chris@0 127 /* protected function getNextFromHeaders(ResponseInterface $response) {
Chris@0 128 $headers = $response->getHeader('Link');
Chris@0 129 foreach ($headers as $header) {
Chris@0 130 $matches = array();
Chris@0 131 preg_match('/^<(.*)>; rel="next"$/', $header, $matches);
Chris@0 132 if (!empty($matches) && !empty($matches[1])) {
Chris@0 133 return $matches[1];
Chris@0 134 }
Chris@0 135 }
Chris@0 136 return FALSE;
Chris@0 137 }
Chris@0 138 */
Chris@0 139 }