Chris@148
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@148
|
2
|
Chris@148
|
3 /*
|
Chris@148
|
4 Sonic Visualiser
|
Chris@148
|
5 An audio file viewer and annotation editor.
|
Chris@148
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@148
|
7 This file copyright 2006 Chris Cannam.
|
Chris@148
|
8
|
Chris@148
|
9 This program is free software; you can redistribute it and/or
|
Chris@148
|
10 modify it under the terms of the GNU General Public License as
|
Chris@148
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@148
|
12 License, or (at your option) any later version. See the file
|
Chris@148
|
13 COPYING included with this distribution for more information.
|
Chris@148
|
14 */
|
Chris@148
|
15
|
Chris@148
|
16 #include "MatrixFile.h"
|
Chris@148
|
17 #include "base/TempDirectory.h"
|
Chris@150
|
18 #include "system/System.h"
|
Chris@148
|
19 #include "base/Profiler.h"
|
Chris@148
|
20 #include "base/Exceptions.h"
|
Chris@408
|
21 #include "base/Thread.h"
|
Chris@148
|
22
|
Chris@148
|
23 #include <sys/types.h>
|
Chris@148
|
24 #include <sys/stat.h>
|
Chris@148
|
25 #include <fcntl.h>
|
Chris@148
|
26 #include <unistd.h>
|
Chris@148
|
27
|
Chris@148
|
28 #include <iostream>
|
Chris@148
|
29
|
Chris@148
|
30 #include <cstdio>
|
Chris@148
|
31 #include <cassert>
|
Chris@148
|
32
|
Chris@405
|
33 #include <cstdlib>
|
Chris@405
|
34
|
Chris@148
|
35 #include <QFileInfo>
|
Chris@148
|
36 #include <QDir>
|
Chris@148
|
37
|
Chris@403
|
38 //#define DEBUG_MATRIX_FILE 1
|
Chris@403
|
39 //#define DEBUG_MATRIX_FILE_READ_SET 1
|
Chris@148
|
40
|
Chris@148
|
41 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@153
|
42 #ifndef DEBUG_MATRIX_FILE
|
Chris@148
|
43 #define DEBUG_MATRIX_FILE 1
|
Chris@148
|
44 #endif
|
Chris@153
|
45 #endif
|
Chris@148
|
46
|
Chris@148
|
47 std::map<QString, int> MatrixFile::m_refcount;
|
Chris@148
|
48 QMutex MatrixFile::m_refcountMutex;
|
Chris@148
|
49
|
Chris@148
|
50 MatrixFile::ResizeableBitsetMap MatrixFile::m_columnBitsets;
|
Chris@148
|
51 QMutex MatrixFile::m_columnBitsetWriteMutex;
|
Chris@148
|
52
|
Chris@148
|
53 FileReadThread *MatrixFile::m_readThread = 0;
|
Chris@148
|
54
|
Chris@148
|
55 static size_t totalStorage = 0;
|
Chris@148
|
56 static size_t totalMemory = 0;
|
Chris@148
|
57 static size_t totalCount = 0;
|
Chris@148
|
58
|
Chris@148
|
59 MatrixFile::MatrixFile(QString fileBase, Mode mode,
|
Chris@148
|
60 size_t cellSize, bool eagerCache) :
|
Chris@148
|
61 m_fd(-1),
|
Chris@148
|
62 m_mode(mode),
|
Chris@148
|
63 m_flags(0),
|
Chris@148
|
64 m_fmode(0),
|
Chris@148
|
65 m_cellSize(cellSize),
|
Chris@148
|
66 m_width(0),
|
Chris@148
|
67 m_height(0),
|
Chris@148
|
68 m_headerSize(2 * sizeof(size_t)),
|
Chris@148
|
69 m_defaultCacheWidth(1024),
|
Chris@148
|
70 m_prevX(0),
|
Chris@148
|
71 m_eagerCache(eagerCache),
|
Chris@148
|
72 m_requestToken(-1),
|
Chris@148
|
73 m_spareData(0),
|
Chris@148
|
74 m_columnBitset(0)
|
Chris@148
|
75 {
|
Chris@148
|
76 Profiler profiler("MatrixFile::MatrixFile", true);
|
Chris@148
|
77
|
Chris@148
|
78 if (!m_readThread) {
|
Chris@148
|
79 m_readThread = new FileReadThread;
|
Chris@148
|
80 m_readThread->start();
|
Chris@148
|
81 }
|
Chris@148
|
82
|
Chris@148
|
83 m_cache.data = 0;
|
Chris@148
|
84
|
Chris@148
|
85 QDir tempDir(TempDirectory::getInstance()->getPath());
|
Chris@148
|
86 QString fileName(tempDir.filePath(QString("%1.mfc").arg(fileBase)));
|
Chris@148
|
87 bool newFile = !QFileInfo(fileName).exists();
|
Chris@148
|
88
|
Chris@148
|
89 if (newFile && m_mode == ReadOnly) {
|
Chris@148
|
90 std::cerr << "ERROR: MatrixFile::MatrixFile: Read-only mode "
|
Chris@148
|
91 << "specified, but cache file does not exist" << std::endl;
|
Chris@148
|
92 throw FileNotFound(fileName);
|
Chris@148
|
93 }
|
Chris@148
|
94
|
Chris@148
|
95 if (!newFile && m_mode == ReadWrite) {
|
Chris@148
|
96 std::cerr << "Note: MatrixFile::MatrixFile: Read/write mode "
|
Chris@148
|
97 << "specified, but file already exists; falling back to "
|
Chris@148
|
98 << "read-only mode" << std::endl;
|
Chris@148
|
99 m_mode = ReadOnly;
|
Chris@148
|
100 }
|
Chris@148
|
101
|
Chris@148
|
102 if (!eagerCache && m_mode == ReadOnly) {
|
Chris@148
|
103 std::cerr << "WARNING: MatrixFile::MatrixFile: Eager cacheing not "
|
Chris@148
|
104 << "specified, but file is open in read-only mode -- cache "
|
Chris@148
|
105 << "will not be used" << std::endl;
|
Chris@148
|
106 }
|
Chris@148
|
107
|
Chris@148
|
108 m_flags = 0;
|
Chris@148
|
109 m_fmode = S_IRUSR | S_IWUSR;
|
Chris@148
|
110
|
Chris@148
|
111 if (m_mode == ReadWrite) {
|
Chris@148
|
112 m_flags = O_RDWR | O_CREAT;
|
Chris@148
|
113 } else {
|
Chris@148
|
114 m_flags = O_RDONLY;
|
Chris@148
|
115 }
|
Chris@148
|
116
|
Chris@233
|
117 #ifdef _WIN32
|
Chris@233
|
118 m_flags |= O_BINARY;
|
Chris@233
|
119 #endif
|
Chris@233
|
120
|
Chris@148
|
121 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
122 std::cerr << "MatrixFile::MatrixFile: opening " << fileName.toStdString() << "..." << std::endl;
|
Chris@148
|
123 #endif
|
Chris@148
|
124
|
Chris@148
|
125 if ((m_fd = ::open(fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
|
Chris@148
|
126 ::perror("Open failed");
|
Chris@148
|
127 std::cerr << "ERROR: MatrixFile::MatrixFile: "
|
Chris@148
|
128 << "Failed to open cache file \""
|
Chris@148
|
129 << fileName.toStdString() << "\"";
|
Chris@148
|
130 if (m_mode == ReadWrite) std::cerr << " for writing";
|
Chris@148
|
131 std::cerr << std::endl;
|
Chris@148
|
132 throw FailedToOpenFile(fileName);
|
Chris@148
|
133 }
|
Chris@148
|
134
|
Chris@148
|
135 if (newFile) {
|
Chris@148
|
136 resize(0, 0); // write header
|
Chris@148
|
137 } else {
|
Chris@148
|
138 size_t header[2];
|
Chris@148
|
139 if (::read(m_fd, header, 2 * sizeof(size_t)) < 0) {
|
Chris@236
|
140 ::perror("MatrixFile::MatrixFile: read failed");
|
Chris@148
|
141 std::cerr << "ERROR: MatrixFile::MatrixFile: "
|
Chris@148
|
142 << "Failed to read header (fd " << m_fd << ", file \""
|
Chris@148
|
143 << fileName.toStdString() << "\")" << std::endl;
|
Chris@148
|
144 throw FileReadFailed(fileName);
|
Chris@148
|
145 }
|
Chris@148
|
146 m_width = header[0];
|
Chris@148
|
147 m_height = header[1];
|
Chris@148
|
148 seekTo(0, 0);
|
Chris@148
|
149 }
|
Chris@148
|
150
|
Chris@148
|
151 m_fileName = fileName;
|
Chris@148
|
152
|
Chris@408
|
153 {
|
Chris@408
|
154 MutexLocker locker
|
Chris@408
|
155 (&m_columnBitsetWriteMutex,
|
Chris@408
|
156 "MatrixFile::MatrixFile::m_columnBitsetWriteMutex");
|
Chris@148
|
157
|
Chris@408
|
158 if (m_columnBitsets.find(m_fileName) == m_columnBitsets.end()) {
|
Chris@408
|
159 m_columnBitsets[m_fileName] = new ResizeableBitset;
|
Chris@408
|
160 }
|
Chris@408
|
161 m_columnBitset = m_columnBitsets[m_fileName];
|
Chris@148
|
162 }
|
Chris@148
|
163
|
Chris@408
|
164 MutexLocker locker(&m_refcountMutex,
|
Chris@408
|
165 "MatrixFile::MatrixFile::m_refcountMutex");
|
Chris@148
|
166 ++m_refcount[fileName];
|
Chris@148
|
167
|
Chris@259
|
168 // std::cerr << "MatrixFile(" << this << "): fd " << m_fd << ", file " << fileName.toStdString() << ", ref " << m_refcount[fileName] << std::endl;
|
Chris@148
|
169
|
Chris@148
|
170 // std::cerr << "MatrixFile::MatrixFile: Done, size is " << "(" << m_width << ", " << m_height << ")" << std::endl;
|
Chris@148
|
171
|
Chris@148
|
172 ++totalCount;
|
Chris@148
|
173
|
Chris@148
|
174 }
|
Chris@148
|
175
|
Chris@148
|
176 MatrixFile::~MatrixFile()
|
Chris@148
|
177 {
|
Chris@148
|
178 char *requestData = 0;
|
Chris@148
|
179
|
Chris@148
|
180 if (m_requestToken >= 0) {
|
Chris@148
|
181 FileReadThread::Request request;
|
Chris@148
|
182 if (m_readThread->getRequest(m_requestToken, request)) {
|
Chris@148
|
183 requestData = request.data;
|
Chris@148
|
184 }
|
Chris@148
|
185 m_readThread->cancel(m_requestToken);
|
Chris@148
|
186 }
|
Chris@148
|
187
|
Chris@148
|
188 if (requestData) free(requestData);
|
Chris@148
|
189 if (m_cache.data) free(m_cache.data);
|
Chris@148
|
190 if (m_spareData) free(m_spareData);
|
Chris@148
|
191
|
Chris@148
|
192 if (m_fd >= 0) {
|
Chris@148
|
193 if (::close(m_fd) < 0) {
|
Chris@148
|
194 ::perror("MatrixFile::~MatrixFile: close failed");
|
Chris@148
|
195 }
|
Chris@148
|
196 }
|
Chris@148
|
197
|
Chris@148
|
198 if (m_fileName != "") {
|
Chris@148
|
199
|
Chris@408
|
200 MutexLocker locker(&m_refcountMutex,
|
Chris@408
|
201 "MatrixFile::~MatrixFile::m_refcountMutex");
|
Chris@148
|
202
|
Chris@148
|
203 if (--m_refcount[m_fileName] == 0) {
|
Chris@148
|
204
|
Chris@148
|
205 if (::unlink(m_fileName.toLocal8Bit())) {
|
Chris@259
|
206 // ::perror("Unlink failed");
|
Chris@259
|
207 // std::cerr << "WARNING: MatrixFile::~MatrixFile: reference count reached 0, but failed to unlink file \"" << m_fileName.toStdString() << "\"" << std::endl;
|
Chris@148
|
208 } else {
|
Chris@259
|
209 // std::cerr << "deleted " << m_fileName.toStdString() << std::endl;
|
Chris@148
|
210 }
|
Chris@148
|
211
|
Chris@408
|
212 MutexLocker locker2
|
Chris@408
|
213 (&m_columnBitsetWriteMutex,
|
Chris@408
|
214 "MatrixFile::~MatrixFile::m_columnBitsetWriteMutex");
|
Chris@148
|
215 m_columnBitsets.erase(m_fileName);
|
Chris@148
|
216 delete m_columnBitset;
|
Chris@148
|
217 }
|
Chris@148
|
218 }
|
Chris@148
|
219
|
Chris@148
|
220 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
|
Chris@148
|
221 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
|
Chris@148
|
222 totalCount --;
|
Chris@148
|
223
|
Chris@259
|
224 // std::cerr << "MatrixFile::~MatrixFile: " << std::endl;
|
Chris@259
|
225 // std::cerr << "Total storage now " << totalStorage/1024 << "K, theoretical max memory "
|
Chris@259
|
226 // << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
|
Chris@148
|
227
|
Chris@148
|
228 }
|
Chris@148
|
229
|
Chris@148
|
230 void
|
Chris@148
|
231 MatrixFile::resize(size_t w, size_t h)
|
Chris@148
|
232 {
|
Chris@148
|
233 Profiler profiler("MatrixFile::resize", true);
|
Chris@148
|
234
|
Chris@148
|
235 assert(m_mode == ReadWrite);
|
Chris@148
|
236
|
Chris@408
|
237 MutexLocker locker(&m_fdMutex, "MatrixFile::resize::m_fdMutex");
|
Chris@148
|
238
|
Chris@148
|
239 totalStorage -= (m_headerSize + (m_width * m_height * m_cellSize));
|
Chris@148
|
240 totalMemory -= (2 * m_defaultCacheWidth * m_height * m_cellSize);
|
Chris@148
|
241
|
Chris@148
|
242 off_t off = m_headerSize + (w * h * m_cellSize);
|
Chris@148
|
243
|
Chris@148
|
244 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
245 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): resizing file" << std::endl;
|
Chris@148
|
246 #endif
|
Chris@148
|
247
|
Chris@148
|
248 if (w * h < m_width * m_height) {
|
Chris@148
|
249 if (::ftruncate(m_fd, off) < 0) {
|
Chris@148
|
250 ::perror("WARNING: MatrixFile::resize: ftruncate failed");
|
Chris@148
|
251 throw FileOperationFailed(m_fileName, "ftruncate");
|
Chris@148
|
252 }
|
Chris@148
|
253 }
|
Chris@148
|
254
|
Chris@148
|
255 m_width = 0;
|
Chris@148
|
256 m_height = 0;
|
Chris@148
|
257
|
Chris@148
|
258 if (::lseek(m_fd, 0, SEEK_SET) == (off_t)-1) {
|
Chris@148
|
259 ::perror("ERROR: MatrixFile::resize: Seek to write header failed");
|
Chris@148
|
260 throw FileOperationFailed(m_fileName, "lseek");
|
Chris@148
|
261 }
|
Chris@148
|
262
|
Chris@148
|
263 size_t header[2];
|
Chris@148
|
264 header[0] = w;
|
Chris@148
|
265 header[1] = h;
|
Chris@148
|
266 if (::write(m_fd, header, 2 * sizeof(size_t)) != 2 * sizeof(size_t)) {
|
Chris@148
|
267 ::perror("ERROR: MatrixFile::resize: Failed to write header");
|
Chris@148
|
268 throw FileOperationFailed(m_fileName, "write");
|
Chris@148
|
269 }
|
Chris@148
|
270
|
Chris@148
|
271 if (w > 0 && m_defaultCacheWidth > w) {
|
Chris@148
|
272 m_defaultCacheWidth = w;
|
Chris@148
|
273 }
|
Chris@148
|
274
|
Chris@411
|
275 //!!! static size_t maxCacheMB = 16;
|
Chris@411
|
276 static size_t maxCacheMB = 4;
|
Chris@148
|
277 if (2 * m_defaultCacheWidth * h * m_cellSize > maxCacheMB * 1024 * 1024) { //!!!
|
Chris@148
|
278 m_defaultCacheWidth = (maxCacheMB * 1024 * 1024) / (2 * h * m_cellSize);
|
Chris@148
|
279 if (m_defaultCacheWidth < 16) m_defaultCacheWidth = 16;
|
Chris@148
|
280 }
|
Chris@148
|
281
|
Chris@148
|
282 if (m_columnBitset) {
|
Chris@408
|
283 MutexLocker locker(&m_columnBitsetWriteMutex,
|
Chris@408
|
284 "MatrixFile::resize::m_columnBitsetWriteMutex");
|
Chris@148
|
285 m_columnBitset->resize(w);
|
Chris@148
|
286 }
|
Chris@148
|
287
|
Chris@148
|
288 if (m_cache.data) {
|
Chris@148
|
289 free(m_cache.data);
|
Chris@148
|
290 m_cache.data = 0;
|
Chris@148
|
291 }
|
Chris@148
|
292
|
Chris@148
|
293 if (m_spareData) {
|
Chris@148
|
294 free(m_spareData);
|
Chris@148
|
295 m_spareData = 0;
|
Chris@148
|
296 }
|
Chris@148
|
297
|
Chris@148
|
298 m_width = w;
|
Chris@148
|
299 m_height = h;
|
Chris@148
|
300
|
Chris@148
|
301 totalStorage += (m_headerSize + (m_width * m_height * m_cellSize));
|
Chris@148
|
302 totalMemory += (2 * m_defaultCacheWidth * m_height * m_cellSize);
|
Chris@148
|
303
|
Chris@148
|
304 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
305 std::cerr << "MatrixFile::resize(" << w << ", " << h << "): cache width "
|
Chris@148
|
306 << m_defaultCacheWidth << ", storage "
|
Chris@148
|
307 << (m_headerSize + w * h * m_cellSize) << ", mem "
|
Chris@148
|
308 << (2 * h * m_defaultCacheWidth * m_cellSize) << std::endl;
|
Chris@148
|
309
|
Chris@148
|
310 std::cerr << "Total storage " << totalStorage/1024 << "K, theoretical max memory "
|
Chris@148
|
311 << totalMemory/1024 << "K in " << totalCount << " instances" << std::endl;
|
Chris@148
|
312 #endif
|
Chris@148
|
313
|
Chris@148
|
314 seekTo(0, 0);
|
Chris@148
|
315 }
|
Chris@148
|
316
|
Chris@148
|
317 void
|
Chris@148
|
318 MatrixFile::reset()
|
Chris@148
|
319 {
|
Chris@148
|
320 Profiler profiler("MatrixFile::reset", true);
|
Chris@148
|
321
|
Chris@148
|
322 assert (m_mode == ReadWrite);
|
Chris@148
|
323
|
Chris@148
|
324 if (m_eagerCache) {
|
Chris@148
|
325 void *emptyCol = calloc(m_height, m_cellSize);
|
Chris@148
|
326 for (size_t x = 0; x < m_width; ++x) setColumnAt(x, emptyCol);
|
Chris@148
|
327 free(emptyCol);
|
Chris@148
|
328 }
|
Chris@148
|
329
|
Chris@148
|
330 if (m_columnBitset) {
|
Chris@408
|
331 MutexLocker locker(&m_columnBitsetWriteMutex,
|
Chris@408
|
332 "MatrixFile::reset::m_columnBitsetWriteMutex");
|
Chris@148
|
333 m_columnBitset->resize(m_width);
|
Chris@148
|
334 }
|
Chris@148
|
335 }
|
Chris@148
|
336
|
Chris@148
|
337 void
|
Chris@148
|
338 MatrixFile::getColumnAt(size_t x, void *data)
|
Chris@148
|
339 {
|
Chris@408
|
340 Profiler profiler("MatrixFile::getColumnAt");
|
Chris@148
|
341
|
Chris@148
|
342 // assert(haveSetColumnAt(x));
|
Chris@148
|
343
|
Chris@148
|
344 if (getFromCache(x, 0, m_height, data)) return;
|
Chris@148
|
345
|
Chris@408
|
346 Profiler profiler2("MatrixFile::getColumnAt (uncached)");
|
Chris@148
|
347
|
Chris@148
|
348 ssize_t r = 0;
|
Chris@148
|
349
|
Chris@148
|
350 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
351 std::cerr << "MatrixFile::getColumnAt(" << x << ")"
|
Chris@148
|
352 << ": reading the slow way";
|
Chris@148
|
353
|
Chris@148
|
354 if (m_requestToken >= 0 &&
|
Chris@148
|
355 x >= m_requestingX &&
|
Chris@148
|
356 x < m_requestingX + m_requestingWidth) {
|
Chris@148
|
357
|
Chris@148
|
358 std::cerr << " (awaiting " << m_requestingX << ", " << m_requestingWidth << " from disk)";
|
Chris@148
|
359 }
|
Chris@148
|
360
|
Chris@148
|
361 std::cerr << std::endl;
|
Chris@148
|
362 #endif
|
Chris@148
|
363
|
Chris@408
|
364 {
|
Chris@408
|
365 MutexLocker locker(&m_fdMutex, "MatrixFile::getColumnAt::m_fdMutex");
|
Chris@148
|
366
|
Chris@408
|
367 if (seekTo(x, 0)) {
|
Chris@408
|
368 r = ::read(m_fd, data, m_height * m_cellSize);
|
Chris@408
|
369 }
|
Chris@148
|
370 }
|
Chris@148
|
371
|
Chris@148
|
372 if (r < 0) {
|
Chris@148
|
373 ::perror("MatrixFile::getColumnAt: read failed");
|
Chris@236
|
374 std::cerr << "ERROR: MatrixFile::getColumnAt: "
|
Chris@236
|
375 << "Failed to read column " << x << " (height " << m_height << ", cell size " << m_cellSize << ", fd " << m_fd << ", file \""
|
Chris@236
|
376 << m_fileName.toStdString() << "\")" << std::endl;
|
Chris@148
|
377 throw FileReadFailed(m_fileName);
|
Chris@148
|
378 }
|
Chris@148
|
379
|
Chris@148
|
380 return;
|
Chris@148
|
381 }
|
Chris@148
|
382
|
Chris@148
|
383 bool
|
Chris@148
|
384 MatrixFile::getFromCache(size_t x, size_t ystart, size_t ycount, void *data)
|
Chris@148
|
385 {
|
Chris@408
|
386 bool fail = false;
|
Chris@408
|
387 bool primeLeft = false;
|
Chris@148
|
388
|
Chris@408
|
389 {
|
Chris@408
|
390 MutexLocker locker(&m_cacheMutex,
|
Chris@408
|
391 "MatrixFile::getFromCache::m_cacheMutex");
|
Chris@408
|
392
|
Chris@408
|
393 if (!m_cache.data || x < m_cache.x || x >= m_cache.x + m_cache.width) {
|
Chris@408
|
394 fail = true;
|
Chris@408
|
395 primeLeft = (m_cache.data && x < m_cache.x);
|
Chris@408
|
396 } else {
|
Chris@408
|
397 memcpy(data,
|
Chris@408
|
398 m_cache.data + m_cellSize * ((x - m_cache.x) * m_height + ystart),
|
Chris@408
|
399 ycount * m_cellSize);
|
Chris@408
|
400 }
|
Chris@408
|
401 }
|
Chris@408
|
402
|
Chris@408
|
403 if (fail) {
|
Chris@408
|
404 primeCache(x, primeLeft); // this doesn't take effect until a later callback
|
Chris@148
|
405 m_prevX = x;
|
Chris@148
|
406 return false;
|
Chris@148
|
407 }
|
Chris@148
|
408
|
Chris@148
|
409 if (m_cache.x > 0 && x < m_prevX && x < m_cache.x + m_cache.width/4) {
|
Chris@148
|
410 primeCache(x, true);
|
Chris@148
|
411 }
|
Chris@148
|
412
|
Chris@148
|
413 if (m_cache.x + m_cache.width < m_width &&
|
Chris@148
|
414 x > m_prevX &&
|
Chris@148
|
415 x > m_cache.x + (m_cache.width * 3) / 4) {
|
Chris@148
|
416 primeCache(x, false);
|
Chris@148
|
417 }
|
Chris@148
|
418
|
Chris@148
|
419 m_prevX = x;
|
Chris@148
|
420 return true;
|
Chris@148
|
421 }
|
Chris@148
|
422
|
Chris@148
|
423 void
|
Chris@148
|
424 MatrixFile::setColumnAt(size_t x, const void *data)
|
Chris@148
|
425 {
|
Chris@148
|
426 assert(m_mode == ReadWrite);
|
Chris@148
|
427
|
Chris@148
|
428 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@411
|
429 // std::cerr << "MatrixFile::setColumnAt(" << x << ")" << std::endl;
|
Chris@411
|
430 std::cerr << ".";
|
Chris@148
|
431 #endif
|
Chris@148
|
432
|
Chris@148
|
433 ssize_t w = 0;
|
Chris@148
|
434 bool seekFailed = false;
|
Chris@408
|
435
|
Chris@408
|
436 {
|
Chris@408
|
437 MutexLocker locker(&m_fdMutex, "MatrixFile::setColumnAt::m_fdMutex");
|
Chris@148
|
438
|
Chris@408
|
439 if (seekTo(x, 0)) {
|
Chris@408
|
440 w = ::write(m_fd, data, m_height * m_cellSize);
|
Chris@408
|
441 } else {
|
Chris@408
|
442 seekFailed = true;
|
Chris@408
|
443 }
|
Chris@148
|
444 }
|
Chris@148
|
445
|
Chris@148
|
446 if (!seekFailed && w != ssize_t(m_height * m_cellSize)) {
|
Chris@148
|
447 ::perror("WARNING: MatrixFile::setColumnAt: write failed");
|
Chris@148
|
448 throw FileOperationFailed(m_fileName, "write");
|
Chris@148
|
449 } else if (seekFailed) {
|
Chris@148
|
450 throw FileOperationFailed(m_fileName, "seek");
|
Chris@148
|
451 } else {
|
Chris@408
|
452 MutexLocker locker
|
Chris@408
|
453 (&m_columnBitsetWriteMutex,
|
Chris@408
|
454 "MatrixFile::setColumnAt::m_columnBitsetWriteMutex");
|
Chris@148
|
455 m_columnBitset->set(x);
|
Chris@148
|
456 }
|
Chris@148
|
457 }
|
Chris@148
|
458
|
Chris@148
|
459 void
|
Chris@148
|
460 MatrixFile::suspend()
|
Chris@148
|
461 {
|
Chris@408
|
462 MutexLocker locker(&m_fdMutex, "MatrixFile::suspend::m_fdMutex");
|
Chris@408
|
463 MutexLocker locker2(&m_cacheMutex, "MatrixFile::suspend::m_cacheMutex");
|
Chris@148
|
464
|
Chris@148
|
465 if (m_fd < 0) return; // already suspended
|
Chris@148
|
466
|
Chris@148
|
467 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
468 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::suspend(): fd was " << m_fd << std::endl;
|
Chris@148
|
469 #endif
|
Chris@148
|
470
|
Chris@148
|
471 if (m_requestToken >= 0) {
|
Chris@148
|
472 void *data = 0;
|
Chris@148
|
473 FileReadThread::Request request;
|
Chris@148
|
474 if (m_readThread->getRequest(m_requestToken, request)) {
|
Chris@148
|
475 data = request.data;
|
Chris@148
|
476 }
|
Chris@148
|
477 m_readThread->cancel(m_requestToken);
|
Chris@148
|
478 if (data) free(data);
|
Chris@148
|
479 m_requestToken = -1;
|
Chris@148
|
480 }
|
Chris@148
|
481
|
Chris@148
|
482 if (m_cache.data) {
|
Chris@148
|
483 free(m_cache.data);
|
Chris@148
|
484 m_cache.data = 0;
|
Chris@148
|
485 }
|
Chris@148
|
486
|
Chris@148
|
487 if (m_spareData) {
|
Chris@148
|
488 free(m_spareData);
|
Chris@148
|
489 m_spareData = 0;
|
Chris@148
|
490 }
|
Chris@148
|
491
|
Chris@148
|
492 if (::close(m_fd) < 0) {
|
Chris@148
|
493 ::perror("WARNING: MatrixFile::suspend: close failed");
|
Chris@148
|
494 throw FileOperationFailed(m_fileName, "close");
|
Chris@148
|
495 }
|
Chris@148
|
496
|
Chris@148
|
497 m_fd = -1;
|
Chris@148
|
498 }
|
Chris@148
|
499
|
Chris@148
|
500 void
|
Chris@148
|
501 MatrixFile::resume()
|
Chris@148
|
502 {
|
Chris@148
|
503 if (m_fd >= 0) return;
|
Chris@148
|
504
|
Chris@148
|
505 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
506 std::cerr << "MatrixFile(" << this << ")::resume()" << std::endl;
|
Chris@148
|
507 #endif
|
Chris@148
|
508
|
Chris@148
|
509 if ((m_fd = ::open(m_fileName.toLocal8Bit(), m_flags, m_fmode)) < 0) {
|
Chris@148
|
510 ::perror("Open failed");
|
Chris@148
|
511 std::cerr << "ERROR: MatrixFile::resume: "
|
Chris@148
|
512 << "Failed to open cache file \""
|
Chris@148
|
513 << m_fileName.toStdString() << "\"";
|
Chris@148
|
514 if (m_mode == ReadWrite) std::cerr << " for writing";
|
Chris@148
|
515 std::cerr << std::endl;
|
Chris@148
|
516 throw FailedToOpenFile(m_fileName);
|
Chris@148
|
517 }
|
Chris@148
|
518
|
Chris@148
|
519 std::cerr << "MatrixFile(" << this << ":" << m_fileName.toStdString() << ")::resume(): fd is " << m_fd << std::endl;
|
Chris@148
|
520 }
|
Chris@148
|
521
|
Chris@411
|
522 static int alloc = 0;
|
Chris@411
|
523
|
Chris@148
|
524 void
|
Chris@148
|
525 MatrixFile::primeCache(size_t x, bool goingLeft)
|
Chris@148
|
526 {
|
Chris@148
|
527 // Profiler profiler("MatrixFile::primeCache");
|
Chris@148
|
528
|
Chris@148
|
529 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@148
|
530 std::cerr << "MatrixFile::primeCache(" << x << ", " << goingLeft << ")" << std::endl;
|
Chris@148
|
531 #endif
|
Chris@148
|
532
|
Chris@148
|
533 size_t rx = x;
|
Chris@148
|
534 size_t rw = m_defaultCacheWidth;
|
Chris@148
|
535
|
Chris@148
|
536 size_t left = rw / 3;
|
Chris@148
|
537 if (goingLeft) left = (rw * 2) / 3;
|
Chris@148
|
538
|
Chris@148
|
539 if (rx > left) rx -= left;
|
Chris@148
|
540 else rx = 0;
|
Chris@148
|
541
|
Chris@148
|
542 if (rx + rw > m_width) rw = m_width - rx;
|
Chris@148
|
543
|
Chris@148
|
544 if (!m_eagerCache) {
|
Chris@148
|
545
|
Chris@148
|
546 size_t ti = 0;
|
Chris@148
|
547
|
Chris@148
|
548 for (ti = 0; ti < rw; ++ti) {
|
Chris@148
|
549 if (!m_columnBitset->get(rx + ti)) break;
|
Chris@148
|
550 }
|
Chris@148
|
551
|
Chris@148
|
552 #ifdef DEBUG_MATRIX_FILE
|
Chris@148
|
553 if (ti < rw) {
|
Chris@148
|
554 std::cerr << "eagerCache is false and there's a hole at "
|
Chris@148
|
555 << rx + ti << ", reducing rw from " << rw << " to "
|
Chris@148
|
556 << ti << std::endl;
|
Chris@148
|
557 }
|
Chris@148
|
558 #endif
|
Chris@148
|
559
|
Chris@148
|
560 rw = std::min(rw, ti);
|
Chris@148
|
561 if (rw < 10 || rx + rw <= x) return;
|
Chris@148
|
562 }
|
Chris@148
|
563
|
Chris@408
|
564 MutexLocker locker(&m_cacheMutex, "MatrixFile::primeCache::m_cacheMutex");
|
Chris@148
|
565
|
Chris@148
|
566 FileReadThread::Request request;
|
Chris@148
|
567
|
Chris@148
|
568 if (m_requestToken >= 0 &&
|
Chris@148
|
569 m_readThread->getRequest(m_requestToken, request)) {
|
Chris@148
|
570
|
Chris@148
|
571 if (x >= m_requestingX &&
|
Chris@148
|
572 x < m_requestingX + m_requestingWidth) {
|
Chris@148
|
573
|
Chris@148
|
574 if (m_readThread->isReady(m_requestToken)) {
|
Chris@148
|
575
|
Chris@148
|
576 if (!request.successful) {
|
Chris@148
|
577 std::cerr << "ERROR: MatrixFile::primeCache: Last request was unsuccessful" << std::endl;
|
Chris@148
|
578 throw FileReadFailed(m_fileName);
|
Chris@148
|
579 }
|
Chris@148
|
580
|
Chris@148
|
581 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@148
|
582 std::cerr << "last request is ready! (" << m_requestingX << ", "<< m_requestingWidth << ")" << std::endl;
|
Chris@148
|
583 #endif
|
Chris@148
|
584
|
Chris@148
|
585 m_cache.x = (request.start - m_headerSize) / (m_height * m_cellSize);
|
Chris@148
|
586 m_cache.width = request.size / (m_height * m_cellSize);
|
Chris@148
|
587
|
Chris@148
|
588 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@148
|
589 std::cerr << "received last request: actual size is: " << m_cache.x << ", " << m_cache.width << std::endl;
|
Chris@148
|
590 #endif
|
Chris@148
|
591
|
Chris@148
|
592 if (m_cache.data) {
|
Chris@411
|
593 if (m_spareData) {
|
Chris@411
|
594 std::cerr << this << ": Freeing spare data" << std::endl;
|
Chris@411
|
595 free(m_spareData);
|
Chris@411
|
596 }
|
Chris@411
|
597 std::cerr << this << ": Moving old cache data to spare" << std::endl;
|
Chris@148
|
598 m_spareData = m_cache.data;
|
Chris@148
|
599 }
|
Chris@411
|
600 std::cerr << this << ": Moving request data to cache" << std::endl;
|
Chris@148
|
601 m_cache.data = request.data;
|
Chris@148
|
602
|
Chris@148
|
603 m_readThread->done(m_requestToken);
|
Chris@148
|
604 m_requestToken = -1;
|
Chris@148
|
605 }
|
Chris@148
|
606
|
Chris@148
|
607 // already requested something covering this area; wait for it
|
Chris@148
|
608 return;
|
Chris@148
|
609 }
|
Chris@148
|
610
|
Chris@148
|
611 // the current request is no longer of any use
|
Chris@148
|
612 m_readThread->cancel(m_requestToken);
|
Chris@148
|
613
|
Chris@148
|
614 // crude way to avoid leaking the data
|
Chris@148
|
615 while (!m_readThread->isCancelled(m_requestToken)) {
|
Chris@148
|
616 usleep(10000);
|
Chris@148
|
617 }
|
Chris@148
|
618
|
Chris@148
|
619 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@148
|
620 std::cerr << "cancelled " << m_requestToken << std::endl;
|
Chris@148
|
621 #endif
|
Chris@148
|
622
|
Chris@411
|
623 if (m_spareData) {
|
Chris@411
|
624 std::cerr << this << ": Freeing spare data" << std::endl;
|
Chris@411
|
625 free(m_spareData);
|
Chris@411
|
626 }
|
Chris@411
|
627 std::cerr << this << ": Moving request data to spare" << std::endl;
|
Chris@148
|
628 m_spareData = request.data;
|
Chris@148
|
629 m_readThread->done(m_requestToken);
|
Chris@148
|
630
|
Chris@148
|
631 m_requestToken = -1;
|
Chris@148
|
632 }
|
Chris@148
|
633
|
Chris@148
|
634 if (m_fd < 0) {
|
Chris@408
|
635 MutexLocker locker(&m_fdMutex, "MatrixFile::primeCache::m_fdMutex");
|
Chris@148
|
636 if (m_fd < 0) resume();
|
Chris@148
|
637 }
|
Chris@148
|
638
|
Chris@148
|
639 request.fd = m_fd;
|
Chris@148
|
640 request.mutex = &m_fdMutex;
|
Chris@148
|
641 request.start = m_headerSize + rx * m_height * m_cellSize;
|
Chris@148
|
642 request.size = rw * m_height * m_cellSize;
|
Chris@411
|
643
|
Chris@411
|
644 std::cerr << this << ": Moving spare data to request, and resizing to " << rw * m_height * m_cellSize << std::endl;
|
Chris@411
|
645
|
Chris@148
|
646 request.data = (char *)realloc(m_spareData, rw * m_height * m_cellSize);
|
Chris@148
|
647 MUNLOCK(request.data, rw * m_height * m_cellSize);
|
Chris@148
|
648 m_spareData = 0;
|
Chris@148
|
649
|
Chris@148
|
650 m_requestingX = rx;
|
Chris@148
|
651 m_requestingWidth = rw;
|
Chris@148
|
652
|
Chris@148
|
653 int token = m_readThread->request(request);
|
Chris@148
|
654 #ifdef DEBUG_MATRIX_FILE_READ_SET
|
Chris@148
|
655 std::cerr << "MatrixFile::primeCache: request token is "
|
Chris@148
|
656 << token << " (x = [" << rx << "], w = [" << rw << "], left = [" << goingLeft << "])" << std::endl;
|
Chris@148
|
657 #endif
|
Chris@148
|
658 m_requestToken = token;
|
Chris@148
|
659 }
|
Chris@148
|
660
|
Chris@148
|
661 bool
|
Chris@148
|
662 MatrixFile::seekTo(size_t x, size_t y)
|
Chris@148
|
663 {
|
Chris@148
|
664 if (m_fd < 0) resume();
|
Chris@148
|
665
|
Chris@148
|
666 off_t off = m_headerSize + (x * m_height + y) * m_cellSize;
|
Chris@148
|
667
|
Chris@148
|
668 if (::lseek(m_fd, off, SEEK_SET) == (off_t)-1) {
|
Chris@148
|
669 ::perror("Seek failed");
|
Chris@148
|
670 std::cerr << "ERROR: MatrixFile::seekTo(" << x << ", " << y
|
Chris@148
|
671 << ") failed" << std::endl;
|
Chris@148
|
672 return false;
|
Chris@148
|
673 }
|
Chris@148
|
674
|
Chris@148
|
675 return true;
|
Chris@148
|
676 }
|
Chris@148
|
677
|