| 199 |
lars |
1 |
<?php namespace Clockwork\Storage;
|
|
|
2 |
|
|
|
3 |
use Clockwork\Request\Request;
|
|
|
4 |
use Clockwork\Storage\Storage;
|
|
|
5 |
|
|
|
6 |
// File based storage for requests
|
|
|
7 |
class FileStorage extends Storage
|
|
|
8 |
{
|
|
|
9 |
// Path where files are stored
|
|
|
10 |
protected $path;
|
|
|
11 |
|
|
|
12 |
// Metadata expiration time in minutes
|
|
|
13 |
protected $expiration;
|
|
|
14 |
|
|
|
15 |
// Compress the files using gzip
|
|
|
16 |
protected $compress;
|
|
|
17 |
|
|
|
18 |
// Metadata cleanup chance
|
|
|
19 |
protected $cleanupChance = 100;
|
|
|
20 |
|
|
|
21 |
// Index file handle
|
|
|
22 |
protected $indexHandle;
|
|
|
23 |
|
|
|
24 |
// Return new storage, takes path where to store files as argument, throws exception if path is not writable
|
|
|
25 |
public function __construct($path, $dirPermissions = 0700, $expiration = null, $compress = false)
|
|
|
26 |
{
|
|
|
27 |
if (! file_exists($path)) {
|
|
|
28 |
// directory doesn't exist, try to create one
|
|
|
29 |
if (! @mkdir($path, $dirPermissions, true)) {
|
|
|
30 |
throw new \Exception("Directory \"{$path}\" does not exist.");
|
|
|
31 |
}
|
|
|
32 |
|
|
|
33 |
// create default .gitignore, to ignore stored json files
|
|
|
34 |
file_put_contents("{$path}/.gitignore", "*.json\n*.json.gz\nindex\n");
|
|
|
35 |
} elseif (! is_writable($path)) {
|
|
|
36 |
throw new \Exception("Path \"{$path}\" is not writable.");
|
|
|
37 |
}
|
|
|
38 |
|
|
|
39 |
if (! file_exists($indexFile = "{$path}/index")) {
|
|
|
40 |
file_put_contents($indexFile, '');
|
|
|
41 |
} elseif (! is_writable($indexFile)) {
|
|
|
42 |
throw new \Exception("Path \"{$indexFile}\" is not writable.");
|
|
|
43 |
}
|
|
|
44 |
|
|
|
45 |
$this->path = $path;
|
|
|
46 |
$this->expiration = $expiration === null ? 60 * 24 * 7 : $expiration;
|
|
|
47 |
$this->compress = $compress;
|
|
|
48 |
}
|
|
|
49 |
|
|
|
50 |
// Returns all requests
|
|
|
51 |
public function all(Search $search = null)
|
|
|
52 |
{
|
|
|
53 |
return $this->loadRequests($this->searchIndexForward($search));
|
|
|
54 |
}
|
|
|
55 |
|
|
|
56 |
// Return a single request by id
|
|
|
57 |
public function find($id)
|
|
|
58 |
{
|
|
|
59 |
return $this->loadRequest($id);
|
|
|
60 |
}
|
|
|
61 |
|
|
|
62 |
// Return the latest request
|
|
|
63 |
public function latest(Search $search = null)
|
|
|
64 |
{
|
|
|
65 |
$requests = $this->loadRequests($this->searchIndexBackward($search, null, 1));
|
|
|
66 |
return reset($requests);
|
|
|
67 |
}
|
|
|
68 |
|
|
|
69 |
// Return requests received before specified id, optionally limited to specified count
|
|
|
70 |
public function previous($id, $count = null, Search $search = null)
|
|
|
71 |
{
|
|
|
72 |
return $this->loadRequests($this->searchIndexBackward($search, $id, $count));
|
|
|
73 |
}
|
|
|
74 |
|
|
|
75 |
// Return requests received after specified id, optionally limited to specified count
|
|
|
76 |
public function next($id, $count = null, Search $search = null)
|
|
|
77 |
{
|
|
|
78 |
return $this->loadRequests($this->searchIndexForward($search, $id, $count));
|
|
|
79 |
}
|
|
|
80 |
|
|
|
81 |
// Store request, requests are stored in JSON representation in files named <request id>.json in storage path
|
|
|
82 |
public function store(Request $request, $skipIndex = false)
|
|
|
83 |
{
|
|
|
84 |
$path = "{$this->path}/{$request->id}.json";
|
|
|
85 |
$data = @json_encode($request->toArray(), \JSON_PARTIAL_OUTPUT_ON_ERROR);
|
|
|
86 |
|
|
|
87 |
$this->compress
|
|
|
88 |
? file_put_contents("{$path}.gz", gzcompress($data))
|
|
|
89 |
: file_put_contents($path, $data);
|
|
|
90 |
|
|
|
91 |
if (! $skipIndex) $this->updateIndex($request);
|
|
|
92 |
|
|
|
93 |
$this->cleanup();
|
|
|
94 |
}
|
|
|
95 |
|
|
|
96 |
// Update existing request
|
|
|
97 |
public function update(Request $request)
|
|
|
98 |
{
|
|
|
99 |
return $this->store($request, true);
|
|
|
100 |
}
|
|
|
101 |
|
|
|
102 |
// Cleanup old requests
|
|
|
103 |
public function cleanup($force = false)
|
|
|
104 |
{
|
|
|
105 |
if ($this->expiration === false || (! $force && rand(1, $this->cleanupChance) != 1)) return;
|
|
|
106 |
|
|
|
107 |
$this->openIndex('start', true, true); // reopen index with lock
|
|
|
108 |
|
|
|
109 |
$expirationTime = time() - ($this->expiration * 60);
|
|
|
110 |
|
|
|
111 |
$old = $this->searchIndexForward(
|
|
|
112 |
new Search([ 'received' => [ '<' . date('c', $expirationTime) ] ], [ 'stopOnFirstMismatch' => true ])
|
|
|
113 |
);
|
|
|
114 |
|
|
|
115 |
if (! count($old)) return $this->closeIndex(true);
|
|
|
116 |
|
|
|
117 |
$this->readPreviousIndex();
|
|
|
118 |
$this->trimIndex();
|
|
|
119 |
$this->closeIndex(true); // explicitly close index to unlock asap
|
|
|
120 |
|
|
|
121 |
foreach ($old as $id) {
|
|
|
122 |
$path = "{$this->path}/{$id}.json";
|
|
|
123 |
@unlink($this->compress ? "{$path}.gz" : $path);
|
|
|
124 |
}
|
|
|
125 |
}
|
|
|
126 |
|
|
|
127 |
// Load a single request by id from filesystem
|
|
|
128 |
protected function loadRequest($id)
|
|
|
129 |
{
|
|
|
130 |
$path = "{$this->path}/{$id}.json";
|
|
|
131 |
|
|
|
132 |
if (! is_readable($this->compress ? "{$path}.gz" : $path)) return;
|
|
|
133 |
|
|
|
134 |
$data = file_get_contents($this->compress ? "{$path}.gz" : $path);
|
|
|
135 |
|
|
|
136 |
return new Request(json_decode($this->compress ? gzuncompress($data) : $data, true));
|
|
|
137 |
}
|
|
|
138 |
|
|
|
139 |
// Load multiple requests by ids from filesystem
|
|
|
140 |
protected function loadRequests($ids)
|
|
|
141 |
{
|
|
|
142 |
return array_filter(array_map(function ($id) { return $this->loadRequest($id); }, $ids));
|
|
|
143 |
}
|
|
|
144 |
|
|
|
145 |
// Search index backward from specified ID or last record, with optional results count limit
|
|
|
146 |
protected function searchIndexBackward(Search $search = null, $id = null, $count = null)
|
|
|
147 |
{
|
|
|
148 |
return $this->searchIndex('previous', $search, $id, $count);
|
|
|
149 |
}
|
|
|
150 |
|
|
|
151 |
// Search index forward from specified ID or last record, with optional results count limit
|
|
|
152 |
protected function searchIndexForward(Search $search = null, $id = null, $count = null)
|
|
|
153 |
{
|
|
|
154 |
return $this->searchIndex('next', $search, $id, $count);
|
|
|
155 |
}
|
|
|
156 |
|
|
|
157 |
// Search index in specified direction from specified ID or last record, with optional results count limit
|
|
|
158 |
protected function searchIndex($direction, Search $search = null, $id = null, $count = null)
|
|
|
159 |
{
|
|
|
160 |
$this->openIndex($direction == 'previous' ? 'end' : 'start', false, true);
|
|
|
161 |
|
|
|
162 |
if ($id) {
|
|
|
163 |
while ($request = $this->readIndex($direction)) { if ($request->id == $id) break; }
|
|
|
164 |
}
|
|
|
165 |
|
|
|
166 |
$found = [];
|
|
|
167 |
|
|
|
168 |
while ($request = $this->readIndex($direction)) {
|
|
|
169 |
if (! $search || $search->matches($request)) {
|
|
|
170 |
$found[] = $request->id;
|
|
|
171 |
} elseif ($search->stopOnFirstMismatch) {
|
|
|
172 |
break;
|
|
|
173 |
}
|
|
|
174 |
|
|
|
175 |
if ($count && count($found) == $count) break;
|
|
|
176 |
}
|
|
|
177 |
|
|
|
178 |
return $direction == 'next' ? $found : array_reverse($found);
|
|
|
179 |
}
|
|
|
180 |
|
|
|
181 |
// Open index file, optionally lock or move file pointer to the end, existing handle will be returned by default
|
|
|
182 |
protected function openIndex($position = 'start', $lock = false, $force = false)
|
|
|
183 |
{
|
|
|
184 |
if ($this->indexHandle) {
|
|
|
185 |
if (! $force) return;
|
|
|
186 |
$this->closeIndex();
|
|
|
187 |
}
|
|
|
188 |
|
|
|
189 |
$this->indexHandle = fopen("{$this->path}/index", 'r');
|
|
|
190 |
|
|
|
191 |
if ($lock) flock($this->indexHandle, LOCK_EX);
|
|
|
192 |
if ($position == 'end') fseek($this->indexHandle, 0, SEEK_END);
|
|
|
193 |
}
|
|
|
194 |
|
|
|
195 |
// Close index file, optionally unlock
|
|
|
196 |
protected function closeIndex($lock = false)
|
|
|
197 |
{
|
|
|
198 |
if ($lock) flock($this->indexHandle, LOCK_UN);
|
|
|
199 |
fclose($this->indexHandle);
|
|
|
200 |
|
|
|
201 |
$this->indexHandle = null;
|
|
|
202 |
}
|
|
|
203 |
|
|
|
204 |
// Read a line from index in the specified direction (next or previous)
|
|
|
205 |
protected function readIndex($direction)
|
|
|
206 |
{
|
|
|
207 |
return $direction == 'next' ? $this->readNextIndex() : $this->readPreviousIndex();
|
|
|
208 |
}
|
|
|
209 |
|
|
|
210 |
// Read previous line from index
|
|
|
211 |
protected function readPreviousIndex()
|
|
|
212 |
{
|
|
|
213 |
$position = ftell($this->indexHandle) - 1;
|
|
|
214 |
|
|
|
215 |
if ($position <= 0) return;
|
|
|
216 |
|
|
|
217 |
$line = '';
|
|
|
218 |
|
|
|
219 |
// reads 1024B chunks of the file backwards from the current position, until a newline is found or we reach the top
|
|
|
220 |
while ($position > 0) {
|
|
|
221 |
// find next position to read from, make sure we don't read beyond file boundary
|
|
|
222 |
$position -= $chunkSize = min($position, 1024);
|
|
|
223 |
|
|
|
224 |
// read the chunk from the position
|
|
|
225 |
fseek($this->indexHandle, $position);
|
|
|
226 |
$chunk = fread($this->indexHandle, $chunkSize);
|
|
|
227 |
|
|
|
228 |
// if a newline is found, append only the part after the last newline, otherwise we can append the whole chunk
|
|
|
229 |
$line = ($newline = strrpos($chunk, "\n")) === false
|
|
|
230 |
? $chunk . $line : substr($chunk, $newline + 1) . $line;
|
|
|
231 |
|
|
|
232 |
// if a newline was found, fix the position so we read from that newline next time
|
|
|
233 |
if ($newline !== false) $position += $newline + 1;
|
|
|
234 |
|
|
|
235 |
// move file pointer to the correct position (revert fread, apply newline fix)
|
|
|
236 |
fseek($this->indexHandle, $position);
|
|
|
237 |
|
|
|
238 |
// if we reached a newline and put together a non-empty line we are done
|
|
|
239 |
if ($newline !== false) break;
|
|
|
240 |
}
|
|
|
241 |
|
|
|
242 |
return $this->makeRequestFromIndex(str_getcsv($line));
|
|
|
243 |
}
|
|
|
244 |
|
|
|
245 |
// Read next line from index
|
|
|
246 |
protected function readNextIndex()
|
|
|
247 |
{
|
|
|
248 |
if (feof($this->indexHandle)) return;
|
|
|
249 |
|
|
|
250 |
// File pointer is always at the start of the line, call extra fgets to skip current line
|
|
|
251 |
fgets($this->indexHandle);
|
|
|
252 |
$line = fgets($this->indexHandle);
|
|
|
253 |
|
|
|
254 |
// Check if we read an empty line or reached the end of file
|
|
|
255 |
if ($line === false) return;
|
|
|
256 |
|
|
|
257 |
// Reset the file pointer to the start of the read line
|
|
|
258 |
fseek($this->indexHandle, ftell($this->indexHandle) - strlen($line));
|
|
|
259 |
|
|
|
260 |
return $this->makeRequestFromIndex(str_getcsv($line));
|
|
|
261 |
}
|
|
|
262 |
|
|
|
263 |
// Trim index file from beginning to current position (including)
|
|
|
264 |
protected function trimIndex()
|
|
|
265 |
{
|
|
|
266 |
// File pointer is always at the start of the line, call extra fgets to skip current line
|
|
|
267 |
fgets($this->indexHandle);
|
|
|
268 |
|
|
|
269 |
// Read the rest of the index file
|
|
|
270 |
$trimmedLength = filesize("{$this->path}/index") - ftell($this->indexHandle);
|
|
|
271 |
$trimmed = $trimmedLength > 0 ? fread($this->indexHandle, $trimmedLength) : '';
|
|
|
272 |
|
|
|
273 |
// Rewrite the index file with a trimmed version
|
|
|
274 |
file_put_contents("{$this->path}/index", $trimmed);
|
|
|
275 |
}
|
|
|
276 |
|
|
|
277 |
// Create an incomplete request from index data
|
|
|
278 |
protected function makeRequestFromIndex($record)
|
|
|
279 |
{
|
|
|
280 |
$type = isset($record[7]) ? $record[7] : 'response';
|
|
|
281 |
|
|
|
282 |
if ($type == 'command') {
|
|
|
283 |
$nameField = 'commandName';
|
|
|
284 |
} elseif ($type == 'queue-job') {
|
|
|
285 |
$nameField = 'jobName';
|
|
|
286 |
} elseif ($type == 'test') {
|
|
|
287 |
$nameField = 'testName';
|
|
|
288 |
} else {
|
|
|
289 |
$nameField = 'uri';
|
|
|
290 |
}
|
|
|
291 |
|
|
|
292 |
return new Request(array_combine(
|
|
|
293 |
[ 'id', 'time', 'method', $nameField, 'controller', 'responseStatus', 'responseDuration', 'type' ],
|
|
|
294 |
array_slice($record, 0, 8) + [ null, null, null, null, null, null, null, 'response' ]
|
|
|
295 |
));
|
|
|
296 |
}
|
|
|
297 |
|
|
|
298 |
// Update index with a new request
|
|
|
299 |
protected function updateIndex(Request $request)
|
|
|
300 |
{
|
|
|
301 |
$handle = fopen("{$this->path}/index", 'a');
|
|
|
302 |
|
|
|
303 |
if (! $handle) return;
|
|
|
304 |
|
|
|
305 |
if (! flock($handle, LOCK_EX)) return fclose($handle);
|
|
|
306 |
|
|
|
307 |
if ($request->type == 'command') {
|
|
|
308 |
$nameField = 'commandName';
|
|
|
309 |
} elseif ($request->type == 'queue-job') {
|
|
|
310 |
$nameField = 'jobName';
|
|
|
311 |
} elseif ($request->type == 'test') {
|
|
|
312 |
$nameField = 'testName';
|
|
|
313 |
} else {
|
|
|
314 |
$nameField = 'uri';
|
|
|
315 |
}
|
|
|
316 |
|
|
|
317 |
fputcsv($handle, [
|
|
|
318 |
$request->id,
|
|
|
319 |
$request->time,
|
|
|
320 |
$request->method,
|
|
|
321 |
$request->$nameField,
|
|
|
322 |
$request->controller,
|
|
|
323 |
$request->responseStatus,
|
|
|
324 |
$request->getResponseDuration(),
|
|
|
325 |
$request->type
|
|
|
326 |
]);
|
|
|
327 |
|
|
|
328 |
flock($handle, LOCK_UN);
|
|
|
329 |
fclose($handle);
|
|
|
330 |
}
|
|
|
331 |
}
|