Nix 2.26.3
Nix, the purely functional package manager; unstable internal interfaces
 
Loading...
Searching...
No Matches
serialise.hh
Go to the documentation of this file.
1#pragma once
3
4#include <memory>
5#include <type_traits>
6
7#include "types.hh"
8#include "util.hh"
9#include "file-descriptor.hh"
10
11namespace boost::context { struct stack_context; }
12
13namespace nix {
14
15
19struct Sink
20{
21 virtual ~Sink() { }
22 virtual void operator () (std::string_view data) = 0;
23 virtual bool good() { return true; }
24};
25
29struct NullSink : Sink
30{
31 void operator () (std::string_view data) override
32 { }
33};
34
35
36struct FinishSink : virtual Sink
37{
38 virtual void finish() = 0;
39};
40
41
46struct BufferedSink : virtual Sink
47{
48 size_t bufSize, bufPos;
49 std::unique_ptr<char[]> buffer;
50
51 BufferedSink(size_t bufSize = 32 * 1024)
52 : bufSize(bufSize), bufPos(0), buffer(nullptr) { }
53
54 void operator () (std::string_view data) override;
55
56 void flush();
57
58protected:
59
60 virtual void writeUnbuffered(std::string_view data) = 0;
61};
62
63
67struct Source
68{
69 virtual ~Source() { }
70
76 void operator () (char * data, size_t len);
77 void operator () (std::string_view data);
78
84 virtual size_t read(char * data, size_t len) = 0;
85
86 virtual bool good() { return true; }
87
88 void drainInto(Sink & sink);
89
90 std::string drain();
91};
92
93
98struct BufferedSource : Source
99{
100 size_t bufSize, bufPosIn, bufPosOut;
101 std::unique_ptr<char[]> buffer;
102
103 BufferedSource(size_t bufSize = 32 * 1024)
104 : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { }
105
106 size_t read(char * data, size_t len) override;
107
111 bool hasData();
112
113protected:
117 virtual size_t readUnbuffered(char * data, size_t len) = 0;
118};
119
120
124struct FdSink : BufferedSink
125{
126 Descriptor fd;
127 size_t written = 0;
128
129 FdSink() : fd(INVALID_DESCRIPTOR) { }
130 FdSink(Descriptor fd) : fd(fd) { }
131 FdSink(FdSink&&) = default;
132
133 FdSink & operator=(FdSink && s)
134 {
135 flush();
136 fd = s.fd;
137 s.fd = INVALID_DESCRIPTOR;
138 written = s.written;
139 return *this;
140 }
141
142 ~FdSink();
143
144 void writeUnbuffered(std::string_view data) override;
145
146 bool good() override;
147
148private:
149 bool _good = true;
150};
151
152
156struct FdSource : BufferedSource
157{
158 Descriptor fd;
159 size_t read = 0;
160 BackedStringView endOfFileError{"unexpected end-of-file"};
161
162 FdSource() : fd(INVALID_DESCRIPTOR) { }
163 FdSource(Descriptor fd) : fd(fd) { }
164 FdSource(FdSource &&) = default;
165
166 FdSource & operator=(FdSource && s) = default;
167
168 bool good() override;
169
174 bool hasData();
175
176protected:
177 size_t readUnbuffered(char * data, size_t len) override;
178private:
179 bool _good = true;
180};
181
182
186struct StringSink : Sink
187{
188 std::string s;
189 StringSink() { }
190 explicit StringSink(const size_t reservedSize)
191 {
192 s.reserve(reservedSize);
193 };
194 StringSink(std::string && s) : s(std::move(s)) { };
195 void operator () (std::string_view data) override;
196};
197
198
202struct StringSource : Source
203{
204 std::string_view s;
205 size_t pos;
206
207 // NOTE: Prevent unintentional dangling views when an implicit conversion
208 // from std::string -> std::string_view occurs when the string is passed
209 // by rvalue.
210 StringSource(std::string &&) = delete;
211 StringSource(std::string_view s) : s(s), pos(0) { }
212 StringSource(const std::string& str): StringSource(std::string_view(str)) {}
213
214 size_t read(char * data, size_t len) override;
215};
216
217
221struct TeeSink : Sink
222{
223 Sink & sink1, & sink2;
224 TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
225 virtual void operator () (std::string_view data) override
226 {
227 sink1(data);
228 sink2(data);
229 }
230};
231
232
236struct TeeSource : Source
237{
238 Source & orig;
239 Sink & sink;
240 TeeSource(Source & orig, Sink & sink)
241 : orig(orig), sink(sink) { }
242 size_t read(char * data, size_t len) override
243 {
244 size_t n = orig.read(data, len);
245 sink({data, n});
246 return n;
247 }
248};
249
253struct SizedSource : Source
254{
255 Source & orig;
256 size_t remain;
257 SizedSource(Source & orig, size_t size)
258 : orig(orig), remain(size) { }
259 size_t read(char * data, size_t len) override
260 {
261 if (this->remain <= 0) {
262 throw EndOfFile("sized: unexpected end-of-file");
263 }
264 len = std::min(len, this->remain);
265 size_t n = this->orig.read(data, len);
266 this->remain -= n;
267 return n;
268 }
269
273 size_t drainAll()
274 {
275 std::vector<char> buf(8192);
276 size_t sum = 0;
277 while (this->remain > 0) {
278 size_t n = read(buf.data(), buf.size());
279 sum += n;
280 }
281 return sum;
282 }
283};
284
289{
290 uint64_t length = 0;
291
292 void operator () (std::string_view data) override
293 {
294 length += data.size();
295 }
296};
297
301struct LengthSource : Source
302{
303 Source & next;
304
305 LengthSource(Source & next) : next(next)
306 { }
307
308 uint64_t total = 0;
309
310 size_t read(char * data, size_t len) override
311 {
312 auto n = next.read(data, len);
313 total += n;
314 return n;
315 }
316};
317
321struct LambdaSink : Sink
322{
323 typedef std::function<void(std::string_view data)> lambda_t;
324
325 lambda_t lambda;
326
327 LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
328
329 void operator () (std::string_view data) override
330 {
331 lambda(data);
332 }
333};
334
335
339struct LambdaSource : Source
340{
341 typedef std::function<size_t(char *, size_t)> lambda_t;
342
343 lambda_t lambda;
344
345 LambdaSource(const lambda_t & lambda) : lambda(lambda) { }
346
347 size_t read(char * data, size_t len) override
348 {
349 return lambda(data, len);
350 }
351};
352
357struct ChainSource : Source
358{
359 Source & source1, & source2;
360 bool useSecond = false;
361 ChainSource(Source & s1, Source & s2)
362 : source1(s1), source2(s2)
363 { }
364
365 size_t read(char * data, size_t len) override;
366};
367
368std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
369
374std::unique_ptr<Source> sinkToSource(
375 std::function<void(Sink &)> fun,
376 std::function<void()> eof = []() {
377 throw EndOfFile("coroutine has finished");
378 });
379
380
381void writePadding(size_t len, Sink & sink);
382void writeString(std::string_view s, Sink & sink);
383
384inline Sink & operator << (Sink & sink, uint64_t n)
385{
386 unsigned char buf[8];
387 buf[0] = n & 0xff;
388 buf[1] = (n >> 8) & 0xff;
389 buf[2] = (n >> 16) & 0xff;
390 buf[3] = (n >> 24) & 0xff;
391 buf[4] = (n >> 32) & 0xff;
392 buf[5] = (n >> 40) & 0xff;
393 buf[6] = (n >> 48) & 0xff;
394 buf[7] = (unsigned char) (n >> 56) & 0xff;
395 sink({(char *) buf, sizeof(buf)});
396 return sink;
397}
398
399Sink & operator << (Sink & in, const Error & ex);
400Sink & operator << (Sink & sink, std::string_view s);
401Sink & operator << (Sink & sink, const Strings & s);
402Sink & operator << (Sink & sink, const StringSet & s);
403
404
405MakeError(SerialisationError, Error);
406
407
408template<typename T>
409T readNum(Source & source)
410{
411 unsigned char buf[8];
412 source((char *) buf, sizeof(buf));
413
414 auto n = readLittleEndian<uint64_t>(buf);
415
416 if (n > (uint64_t) std::numeric_limits<T>::max())
417 throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
418
419 return (T) n;
420}
421
422
423inline unsigned int readInt(Source & source)
424{
425 return readNum<unsigned int>(source);
426}
427
428
429inline uint64_t readLongLong(Source & source)
430{
431 return readNum<uint64_t>(source);
432}
433
434
435void readPadding(size_t len, Source & source);
436size_t readString(char * buf, size_t max, Source & source);
437std::string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
438template<class T> T readStrings(Source & source);
439
440Source & operator >> (Source & in, std::string & s);
441
442template<typename T>
443Source & operator >> (Source & in, T & n)
444{
445 n = readNum<T>(in);
446 return in;
447}
448
449template<typename T>
450Source & operator >> (Source & in, bool & b)
451{
452 b = readNum<uint64_t>(in);
453 return in;
454}
455
456Error readError(Source & source);
457
458
462struct StreamToSourceAdapter : Source
463{
464 std::shared_ptr<std::basic_istream<char>> istream;
465
466 StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
467 : istream(istream)
468 { }
469
470 size_t read(char * data, size_t len) override
471 {
472 if (!istream->read(data, len)) {
473 if (istream->eof()) {
474 if (istream->gcount() == 0)
475 throw EndOfFile("end of file");
476 } else
477 throw Error("I/O error in StreamToSourceAdapter");
478 }
479 return istream->gcount();
480 }
481};
482
483
492struct FramedSource : Source
493{
494 Source & from;
495 bool eof = false;
496 std::vector<char> pending;
497 size_t pos = 0;
498
499 FramedSource(Source & from) : from(from)
500 { }
501
502 ~FramedSource()
503 {
504 try {
505 if (!eof) {
506 while (true) {
507 auto n = readInt(from);
508 if (!n) break;
509 std::vector<char> data(n);
510 from(data.data(), n);
511 }
512 }
513 } catch (...) {
514 ignoreExceptionInDestructor();
515 }
516 }
517
518 size_t read(char * data, size_t len) override
519 {
520 if (eof) throw EndOfFile("reached end of FramedSource");
521
522 if (pos >= pending.size()) {
523 size_t len = readInt(from);
524 if (!len) {
525 eof = true;
526 return 0;
527 }
528 pending = std::vector<char>(len);
529 pos = 0;
530 from(pending.data(), len);
531 }
532
533 auto n = std::min(len, pending.size() - pos);
534 memcpy(data, pending.data() + pos, n);
535 pos += n;
536 return n;
537 }
538};
539
546struct FramedSink : nix::BufferedSink
547{
548 BufferedSink & to;
549 std::function<void()> checkError;
550
551 FramedSink(BufferedSink & to, std::function<void()> && checkError)
552 : to(to), checkError(checkError)
553 { }
554
555 ~FramedSink()
556 {
557 try {
558 to << 0;
559 to.flush();
560 } catch (...) {
561 ignoreExceptionInDestructor();
562 }
563 }
564
565 void writeUnbuffered(std::string_view data) override
566 {
567 /* Don't send more data if an error has occured. */
568 checkError();
569
570 to << data.size();
571 to(data);
572 };
573};
574
575}
Definition types.hh:65
int Descriptor
Definition file-descriptor.hh:20
uint32_t size_t
Definition lexer.l:6336
char
Definition lexer.l:3739
return s
Definition lexer.l:459
std::function< void(Sink &)> fun
Definition lexer.l:3485
std::ostream & str
Definition lexer.l:1728
const std::string_view & name
Definition lexer.l:1709
std::variant< std::string, std::string_view > data
Definition lexer.l:177
Definition serialise.hh:47
virtual size_t readUnbuffered(char *data, size_t len)=0
size_t read(char *data, size_t len) override
Definition serialise.cc:114
bool hasData()
Definition serialise.cc:129
size_t read(char *data, size_t len) override
Definition serialise.cc:454
bool hasData()
Definition serialise.cc:164
size_t readUnbuffered(char *data, size_t len) override
Definition serialise.cc:135
Definition serialise.hh:37
size_t read(char *data, size_t len) override
Definition serialise.hh:518
size_t read(char *data, size_t len) override
Definition serialise.hh:347
Definition serialise.hh:289
size_t read(char *data, size_t len) override
Definition serialise.hh:310
Definition serialise.hh:30
Definition serialise.hh:20
size_t drainAll()
Definition serialise.hh:273
size_t read(char *data, size_t len) override
Definition serialise.hh:259
Definition serialise.hh:68
void operator()(char *data, size_t len)
Definition serialise.cc:78
virtual size_t read(char *data, size_t len)=0
size_t read(char *data, size_t len) override
Definition serialise.hh:470
size_t read(char *data, size_t len) override
Definition serialise.cc:188
size_t read(char *data, size_t len) override
Definition serialise.hh:242
T readLittleEndian(unsigned char *p)
Definition util.hh:125