Chris@0
|
1 <?php
|
Chris@0
|
2
|
Chris@0
|
3 namespace Drupal\migrate_plus\Plugin\migrate\source;
|
Chris@0
|
4
|
Chris@0
|
5 use Drupal\migrate\Plugin\MigrationInterface;
|
Chris@0
|
6 use Drupal\migrate_plus\DataParserPluginInterface;
|
Chris@0
|
7
|
Chris@0
|
8 /**
|
Chris@0
|
9 * Source plugin for retrieving data via URLs.
|
Chris@0
|
10 *
|
Chris@0
|
11 * @MigrateSource(
|
Chris@0
|
12 * id = "url"
|
Chris@0
|
13 * )
|
Chris@0
|
14 */
|
Chris@0
|
15 class Url extends SourcePluginExtension {
|
Chris@0
|
16
|
Chris@0
|
17 /**
|
Chris@0
|
18 * The source URLs to retrieve.
|
Chris@0
|
19 *
|
Chris@0
|
20 * @var array
|
Chris@0
|
21 */
|
Chris@0
|
22 protected $sourceUrls = [];
|
Chris@0
|
23
|
Chris@0
|
24 /**
|
Chris@0
|
25 * The data parser plugin.
|
Chris@0
|
26 *
|
Chris@0
|
27 * @var \Drupal\migrate_plus\DataParserPluginInterface
|
Chris@0
|
28 */
|
Chris@0
|
29 protected $dataParserPlugin;
|
Chris@0
|
30
|
Chris@0
|
31 /**
|
Chris@0
|
32 * {@inheritdoc}
|
Chris@0
|
33 */
|
Chris@0
|
34 public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
|
Chris@0
|
35 if (!is_array($configuration['urls'])) {
|
Chris@0
|
36 $configuration['urls'] = [$configuration['urls']];
|
Chris@0
|
37 }
|
Chris@0
|
38 parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
|
Chris@0
|
39
|
Chris@0
|
40 $this->sourceUrls = $configuration['urls'];
|
Chris@0
|
41
|
Chris@0
|
42 // Set a default Accept header.
|
Chris@0
|
43 /* $this->headers = array_merge(['Accept' => 'application/json'],
|
Chris@0
|
44 $configuration['headers'] ?: []);*/
|
Chris@0
|
45
|
Chris@0
|
46 // See if this is a paged response with next links. If so, add to the source_urls array.
|
Chris@0
|
47 /* foreach ( (array) $configuration['urls'] as $url) {
|
Chris@0
|
48 $this->sourceUrls += $this->getNextLinks($url);
|
Chris@0
|
49 }*/
|
Chris@0
|
50 }
|
Chris@0
|
51
|
Chris@0
|
52 /**
|
Chris@0
|
53 * Return a string representing the source URLs.
|
Chris@0
|
54 *
|
Chris@0
|
55 * @return string
|
Chris@0
|
56 * Comma-separated list of URLs being imported.
|
Chris@0
|
57 */
|
Chris@0
|
58 public function __toString() {
|
Chris@0
|
59 // This could cause a problem when using a lot of urls, may need to hash.
|
Chris@0
|
60 $urls = implode(', ', $this->sourceUrls);
|
Chris@0
|
61 return $urls;
|
Chris@0
|
62 }
|
Chris@0
|
63
|
Chris@0
|
64 /**
|
Chris@0
|
65 * Returns the initialized data parser plugin.
|
Chris@0
|
66 *
|
Chris@0
|
67 * @return \Drupal\migrate_plus\DataParserPluginInterface
|
Chris@0
|
68 * The data parser plugin.
|
Chris@0
|
69 */
|
Chris@0
|
70 public function getDataParserPlugin() {
|
Chris@0
|
71 if (!isset($this->dataParserPlugin)) {
|
Chris@0
|
72 $this->dataParserPlugin = \Drupal::service('plugin.manager.migrate_plus.data_parser')->createInstance($this->configuration['data_parser_plugin'], $this->configuration);
|
Chris@0
|
73 }
|
Chris@0
|
74 return $this->dataParserPlugin;
|
Chris@0
|
75 }
|
Chris@0
|
76
|
Chris@0
|
77 /**
|
Chris@0
|
78 * Creates and returns a filtered Iterator over the documents.
|
Chris@0
|
79 *
|
Chris@0
|
80 * @return \Iterator
|
Chris@0
|
81 * An iterator over the documents providing source rows that match the
|
Chris@0
|
82 * configured item_selector.
|
Chris@0
|
83 */
|
Chris@0
|
84 protected function initializeIterator() {
|
Chris@0
|
85 return $this->getDataParserPlugin();
|
Chris@0
|
86 }
|
Chris@0
|
87
|
Chris@0
|
88 /**
|
Chris@0
|
89 * Collect an array of next links from a paged response.
|
Chris@0
|
90 */
|
Chris@0
|
91 /* protected function getNextLinks($url) {
|
Chris@0
|
92 $urls = array();
|
Chris@0
|
93 $more = TRUE;
|
Chris@0
|
94 while ($more == TRUE) {
|
Chris@0
|
95 $response = $this->dataParserPlugin->getDataFetcher()->getResponse($url);
|
Chris@0
|
96 if ($url = $this->getNextFromHeaders($response)) {
|
Chris@0
|
97 $urls[] = $url;
|
Chris@0
|
98 }
|
Chris@0
|
99 elseif ($url = $this->getNextFromLinks($response)) {
|
Chris@0
|
100 $urls[] = $url;
|
Chris@0
|
101 }
|
Chris@0
|
102 else {
|
Chris@0
|
103 $more = FALSE;
|
Chris@0
|
104 }
|
Chris@0
|
105 }
|
Chris@0
|
106 return $urls;
|
Chris@0
|
107 }
|
Chris@0
|
108 */
|
Chris@0
|
109 /**
|
Chris@0
|
110 * See if the next link is in a 'links' group in the response.
|
Chris@0
|
111 *
|
Chris@0
|
112 * @param \Psr\Http\Message\ResponseInterface $response
|
Chris@0
|
113 */
|
Chris@0
|
114 /* protected function getNextFromLinks(ResponseInterface $response) {
|
Chris@0
|
115 $body = json_decode($response->getBody(), TRUE);
|
Chris@0
|
116 if (!empty($body['links']) && array_key_exists('next', $body['links'])) {
|
Chris@0
|
117 return $body['links']['next'];
|
Chris@0
|
118 }
|
Chris@0
|
119 return FALSE;
|
Chris@0
|
120 }
|
Chris@0
|
121 */
|
Chris@0
|
122 /**
|
Chris@0
|
123 * See if the next link is in the header.
|
Chris@0
|
124 *
|
Chris@0
|
125 * @param \Psr\Http\Message\ResponseInterface $response
|
Chris@0
|
126 */
|
Chris@0
|
127 /* protected function getNextFromHeaders(ResponseInterface $response) {
|
Chris@0
|
128 $headers = $response->getHeader('Link');
|
Chris@0
|
129 foreach ($headers as $header) {
|
Chris@0
|
130 $matches = array();
|
Chris@0
|
131 preg_match('/^<(.*)>; rel="next"$/', $header, $matches);
|
Chris@0
|
132 if (!empty($matches) && !empty($matches[1])) {
|
Chris@0
|
133 return $matches[1];
|
Chris@0
|
134 }
|
Chris@0
|
135 }
|
Chris@0
|
136 return FALSE;
|
Chris@0
|
137 }
|
Chris@0
|
138 */
|
Chris@0
|
139 }
|