projects/guix·blob python-team · refs

285 lines · 6.3 KiB

1#include "serialise.hh"
2#include "util.hh"
3
4#include <cstring>
5#include <cerrno>
6#include <cassert>
7
8namespace nix {
9
10
11BufferedSink::~BufferedSink()
12{
13 /* We can't call flush() here, because C++ for some insane reason
14 doesn't allow you to call virtual methods from a destructor. */
15 assert(!bufPos);
16 delete[] buffer;
17}
18
19
20void BufferedSink::operator () (const unsigned char * data, size_t len)
21{
22 if (!buffer) buffer = new unsigned char[bufSize];
23
24 while (len) {
25 /* Optimisation: bypass the buffer if the data exceeds the
26 buffer size. */
27 if (bufPos + len >= bufSize) {
28 flush();
29 write(data, len);
30 break;
31 }
32 /* Otherwise, copy the bytes to the buffer. Flush the buffer
33 when it's full. */
34 size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
35 memcpy(buffer + bufPos, data, n);
36 data += n; bufPos += n; len -= n;
37 if (bufPos == bufSize) flush();
38 }
39}
40
41
42void BufferedSink::flush()
43{
44 if (bufPos == 0) return;
45 size_t n = bufPos;
46 bufPos = 0; // don't trigger the assert() in ~BufferedSink()
47 write(buffer, n);
48}
49
50
51FdSink::~FdSink()
52{
53 try { flush(); } catch (...) { ignoreException(); }
54}
55
56
57size_t threshold = 256 * 1024 * 1024;
58
59static void warnLargeDump()
60{
61 printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory");
62}
63
64
65void FdSink::write(const unsigned char * data, size_t len)
66{
67 static bool warned = false;
68 if (warn && !warned) {
69 written += len;
70 if (written > threshold) {
71 warnLargeDump();
72 warned = true;
73 }
74 }
75 writeFull(fd, data, len);
76}
77
78
79void Source::operator () (unsigned char * data, size_t len)
80{
81 while (len) {
82 size_t n = read(data, len);
83 data += n; len -= n;
84 }
85}
86
87
88BufferedSource::~BufferedSource()
89{
90 delete[] buffer;
91}
92
93
94size_t BufferedSource::read(unsigned char * data, size_t len)
95{
96 if (!buffer) buffer = new unsigned char[bufSize];
97
98 if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize);
99
100 /* Copy out the data in the buffer. */
101 size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
102 memcpy(data, buffer + bufPosOut, n);
103 bufPosOut += n;
104 if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
105 return n;
106}
107
108
109bool BufferedSource::hasData()
110{
111 return bufPosOut < bufPosIn;
112}
113
114
115size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
116{
117 ssize_t n;
118 do {
119 checkInterrupt();
120 n = ::read(fd, (char *) data, bufSize);
121 } while (n == -1 && errno == EINTR);
122 if (n == -1) throw SysError("reading from file");
123 if (n == 0) throw EndOfFile("unexpected end-of-file");
124 return n;
125}
126
127
128size_t StringSource::read(unsigned char * data, size_t len)
129{
130 if (pos == s.size()) throw EndOfFile("end of string reached");
131 size_t n = s.copy((char *) data, len, pos);
132 pos += n;
133 return n;
134}
135
136
137void writePadding(size_t len, Sink & sink)
138{
139 if (len % 8) {
140 unsigned char zero[8];
141 memset(zero, 0, sizeof(zero));
142 sink(zero, 8 - (len % 8));
143 }
144}
145
146
147void writeInt(unsigned int n, Sink & sink)
148{
149 unsigned char buf[8];
150 memset(buf, 0, sizeof(buf));
151 buf[0] = n & 0xff;
152 buf[1] = (n >> 8) & 0xff;
153 buf[2] = (n >> 16) & 0xff;
154 buf[3] = (n >> 24) & 0xff;
155 sink(buf, sizeof(buf));
156}
157
158
159void writeLongLong(unsigned long long n, Sink & sink)
160{
161 unsigned char buf[8];
162 buf[0] = n & 0xff;
163 buf[1] = (n >> 8) & 0xff;
164 buf[2] = (n >> 16) & 0xff;
165 buf[3] = (n >> 24) & 0xff;
166 buf[4] = (n >> 32) & 0xff;
167 buf[5] = (n >> 40) & 0xff;
168 buf[6] = (n >> 48) & 0xff;
169 buf[7] = (n >> 56) & 0xff;
170 sink(buf, sizeof(buf));
171}
172
173
174void writeString(const unsigned char * buf, size_t len, Sink & sink)
175{
176 writeInt(len, sink);
177 sink(buf, len);
178 writePadding(len, sink);
179}
180
181
182void writeString(const string & s, Sink & sink)
183{
184 writeString((const unsigned char *) s.data(), s.size(), sink);
185}
186
187
188template<class T> void writeStrings(const T & ss, Sink & sink)
189{
190 writeInt(ss.size(), sink);
191 for (auto& i : ss)
192 writeString(i, sink);
193}
194
195template void writeStrings(const Paths & ss, Sink & sink);
196template void writeStrings(const PathSet & ss, Sink & sink);
197
198
199void readPadding(size_t len, Source & source)
200{
201 if (len % 8) {
202 unsigned char zero[8];
203 size_t n = 8 - (len % 8);
204 source(zero, n);
205 for (unsigned int i = 0; i < n; i++)
206 if (zero[i]) throw SerialisationError("non-zero padding");
207 }
208}
209
210
211unsigned int readInt(Source & source)
212{
213 unsigned char buf[8];
214 source(buf, sizeof(buf));
215 if (buf[4] || buf[5] || buf[6] || buf[7])
216 throw SerialisationError("implementation cannot deal with > 32-bit integers");
217 return
218 buf[0] |
219 (buf[1] << 8) |
220 (buf[2] << 16) |
221 (buf[3] << 24);
222}
223
224
225unsigned long long readLongLong(Source & source)
226{
227 unsigned char buf[8];
228 source(buf, sizeof(buf));
229 return
230 ((unsigned long long) buf[0]) |
231 ((unsigned long long) buf[1] << 8) |
232 ((unsigned long long) buf[2] << 16) |
233 ((unsigned long long) buf[3] << 24) |
234 ((unsigned long long) buf[4] << 32) |
235 ((unsigned long long) buf[5] << 40) |
236 ((unsigned long long) buf[6] << 48) |
237 ((unsigned long long) buf[7] << 56);
238}
239
240
241size_t readString(unsigned char * buf, size_t max, Source & source)
242{
243 size_t len = readInt(source);
244 if (len > max) throw Error("string is too long");
245 source(buf, len);
246 readPadding(len, source);
247 return len;
248}
249
250
251string readString(Source & source)
252{
253 size_t len = readInt(source);
254 std::vector<unsigned char> buf(len);
255 source(buf.data(), buf.size());
256 readPadding(buf.size(), source);
257 return string((char *) buf.data(), buf.size());
258}
259
260
261template<class T> T readStrings(Source & source)
262{
263 unsigned int count = readInt(source);
264 T ss;
265 while (count--)
266 ss.insert(ss.end(), readString(source));
267 return ss;
268}
269
270template Paths readStrings(Source & source);
271template PathSet readStrings(Source & source);
272
273
274void StringSink::operator () (const unsigned char * data, size_t len)
275{
276 static bool warned = false;
277 if (!warned && s.size() > threshold) {
278 warnLargeDump();
279 warned = true;
280 }
281 s.append((const char *) data, len);
282}
283
284
285}