projects/guix·blob python-team · refs

273 lines · 8.6 KiB

1#include "config.h"
2
3#include "misc.hh"
4#include "util.hh"
5#include "local-store.hh"
6#include "globals.hh"
7
8#include <cstdlib>
9#include <cstring>
10#include <sys/types.h>
11#include <sys/stat.h>
12#include <unistd.h>
13#include <errno.h>
14#include <stdio.h>
15#include <format>
16
17namespace nix {
18
19
20static void makeWritable(const Path & path)
21{
22 struct stat st;
23 if (lstat(path.c_str(), &st))
24 throw SysError(std::format("getting attributes of path `{}'", path));
25 if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
26 throw SysError(std::format("changing writability of `{}'", path));
27}
28
29
30struct MakeReadOnly
31{
32 Path path;
33 MakeReadOnly(const Path & path) : path(path) { }
34 ~MakeReadOnly()
35 {
36 try {
37 /* This will make the path read-only. */
38 if (path != "") canonicaliseTimestampAndPermissions(path);
39 } catch (...) {
40 ignoreException();
41 }
42 }
43};
44
45
46LocalStore::InodeHash LocalStore::loadInodeHash()
47{
48 printMsg(lvlDebug, "loading hash inodes in memory");
49 InodeHash inodeHash;
50
51 AutoCloseDir dir = opendir(linksDir.c_str());
52 if (!dir) throw SysError(std::format("opening directory `{}'", linksDir));
53
54 struct dirent * dirent;
55 while (errno = 0, dirent = readdir(dir)) { /* sic */
56 checkInterrupt();
57 // We don't care if we hit non-hash files, anything goes
58 inodeHash.insert(dirent->d_ino);
59 }
60 if (errno) throw SysError(std::format("reading directory `{}'", linksDir));
61
62 printMsg(lvlTalkative, std::format("loaded {} hash inodes", inodeHash.size()));
63
64 return inodeHash;
65}
66
67
68Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
69{
70 Strings names;
71
72 AutoCloseDir dir = opendir(path.c_str());
73 if (!dir) throw SysError(std::format("opening directory `{}'", path));
74
75 struct dirent * dirent;
76 while (errno = 0, dirent = readdir(dir)) { /* sic */
77 checkInterrupt();
78
79 if (inodeHash.count(dirent->d_ino)) {
80 printMsg(lvlDebug, std::format("`{}' is already linked", dirent->d_name));
81 continue;
82 }
83
84 string name = dirent->d_name;
85 if (name == "." || name == "..") continue;
86 names.push_back(name);
87 }
88 if (errno) throw SysError(std::format("reading directory `{}'", path));
89
90 return names;
91}
92
93
94void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
95{
96 checkInterrupt();
97
98 struct stat st;
99 if (lstat(path.c_str(), &st))
100 throw SysError(std::format("getting attributes of path `{}'", path));
101
102 if (S_ISDIR(st.st_mode)) {
103 Strings names = readDirectoryIgnoringInodes(path, inodeHash);
104 for (auto& i : names)
105 optimisePath_(stats, path + "/" + i, inodeHash);
106 return;
107 }
108
109 /* We can hard link regular files (and maybe symlinks), but do that only
110 for files larger than some threshold. This avoids adding too many
111 entries to '.links', which would slow down 'removeUnusedLinks' while
112 saving little space. */
113 if (!S_ISREG(st.st_mode) || ((size_t) st.st_size) < deduplicationMinSize)
114 return;
115
116 /* Sometimes SNAFUs can cause files in the store to be
117 modified, in particular when running programs as root under
118 Guix System (example: $fontconfig/var/cache being modified). Skip
119 those files. FIXME: check the modification time. */
120 if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
121 printMsg(lvlError, std::format("skipping suspicious writable file `{}'", path));
122 return;
123 }
124
125 /* This can still happen on top-level files. */
126 if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
127 printMsg(lvlDebug, std::format("`{}' is already linked, with {} other file(s).", path, (st.st_nlink - 2)));
128 return;
129 }
130
131 /* Hash the file. Note that hashPath() returns the hash over the
132 NAR serialisation, which includes the execute bit on the file.
133 Thus, executable and non-executable files with the same
134 contents *won't* be linked (which is good because otherwise the
135 permissions would be screwed up).
136
137 Also note that if `path' is a symlink, then we're hashing the
138 contents of the symlink (i.e. the result of readlink()), not
139 the contents of the target (which may not even exist). */
140 Hash hash = hashPath(htSHA256, path).first;
141 printMsg(lvlDebug, std::format("`{}' has hash `{}'", path, printHash(hash)));
142
143 /* Check if this is a known hash. */
144 Path linkPath = linksDir + "/" + printHash32(hash);
145
146 retry:
147 if (!pathExists(linkPath)) {
148 /* Nope, create a hard link in the links directory. */
149 if (link(path.c_str(), linkPath.c_str()) == 0) {
150 inodeHash.insert(st.st_ino);
151 return;
152 }
153
154 switch (errno) {
155 case EEXIST:
156 /* Fall through if another process created ‘linkPath’ before
157 we did. */
158 break;
159
160 case ENOSPC:
161 /* On ext4, that probably means the directory index is full. When
162 that happens, it's fine to ignore it: we just effectively
163 disable deduplication of this file. */
164 printMsg(lvlInfo, std::format("cannot link `{}' to `{}': {}",
165 linkPath, path, strerror(ENOSPC)));
166 return;
167
168 default:
169 throw SysError(std::format("cannot link `{}' to `{}'", linkPath, path));
170 }
171 }
172
173 /* Yes! We've seen a file with the same contents. Replace the
174 current file with a hard link to that file. */
175 struct stat stLink;
176 if (lstat(linkPath.c_str(), &stLink))
177 throw SysError(std::format("getting attributes of path `{}'", linkPath));
178
179 if (st.st_ino == stLink.st_ino) {
180 printMsg(lvlDebug, std::format("`{}' is already linked to `{}'", path, linkPath));
181 return;
182 }
183
184 if (st.st_size != stLink.st_size) {
185 printMsg(lvlError, std::format("removing corrupted link `{}'", linkPath));
186 unlink(linkPath.c_str());
187 goto retry;
188 }
189
190 printMsg(lvlTalkative, std::format("linking `{}' to `{}'", path, linkPath));
191
192 /* Make the containing directory writable, but only if it's not
193 the store itself (we don't want or need to mess with its
194 permissions). */
195 bool mustToggle = !isStorePath(path);
196 if (mustToggle) makeWritable(dirOf(path));
197
198 /* When we're done, make the directory read-only again and reset
199 its timestamp back to 0. */
200 MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
201
202 Path tempLink = std::format("{}/.tmp-link-{}-{}", settings.nixStore, getpid(), rand());
203
204 if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
205 if (errno == EMLINK) {
206 /* Too many links to the same file (>= 32000 on most file
207 systems). This is likely to happen with empty files.
208 Just shrug and ignore. */
209 if (st.st_size)
210 printMsg(lvlInfo, std::format("`{}' has maximum number of links", linkPath));
211 return;
212 }
213 throw SysError(std::format("cannot link `{}' to `{}'", tempLink, linkPath));
214 }
215
216 /* Atomically replace the old file with the new hard link. */
217 if (rename(tempLink.c_str(), path.c_str()) == -1) {
218 int renameErrno = errno;
219 if (unlink(tempLink.c_str()) == -1)
220 printMsg(lvlError, std::format("unable to unlink `{}'", tempLink));
221 if (renameErrno == EMLINK) {
222 /* Some filesystems generate too many links on the rename,
223 rather than on the original link. (Probably it
224 temporarily increases the st_nlink field before
225 decreasing it again.) */
226 if (st.st_size)
227 printMsg(lvlInfo, std::format("`{}' has maximum number of links", linkPath));
228 return;
229 }
230 throw SysError(std::format("cannot rename `{}' to `{}'", tempLink, path));
231 }
232
233 stats.filesLinked++;
234 stats.bytesFreed += st.st_size;
235 stats.blocksFreed += st.st_blocks;
236}
237
238
239void LocalStore::optimiseStore(OptimiseStats & stats)
240{
241 PathSet paths = queryAllValidPaths();
242 InodeHash inodeHash = loadInodeHash();
243
244 for (auto& i : paths) {
245 addTempRoot(i);
246 if (!isValidPath(i)) continue; /* path was GC'ed, probably */
247 startNest(nest, lvlChatty, std::format("hashing files in `{}'", i));
248 optimisePath_(stats, i, inodeHash);
249 }
250}
251
252void LocalStore::optimiseStore()
253{
254 OptimiseStats stats;
255
256 optimiseStore(stats);
257
258 printMsg(lvlError,
259 std::format("{} freed by hard-linking {} files",
260 showBytes(stats.bytesFreed),
261 stats.filesLinked));
262}
263
264void LocalStore::optimisePath(const Path & path)
265{
266 OptimiseStats stats;
267 InodeHash inodeHash;
268
269 if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
270}
271
272
273}