dsnapshot.backend.rsync source code

1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.backend.rsync;
7 
8 import logger = std.experimental.logger;
9 import std.algorithm : map, filter;
10 import std.datetime : SysTime;
11 
12 import dsnapshot.backend.crypt;
13 import dsnapshot.backend.rsync;
14 import dsnapshot.backend;
15 import dsnapshot.config;
16 import dsnapshot.exception;
17 import dsnapshot.from;
18 import dsnapshot.layout : Layout;
19 import dsnapshot.process;
20 import dsnapshot.types;
21 
22 @safe:
23 
24 final class RsyncBackend : SyncBackend {
25     RsyncConfig conf;
26     RemoteCmd remoteCmd_;
27     /// Error codes ignored when Synchronizing.
28     const(int)[] ignoreRsyncErrorCodes;
29 
30     this(RsyncConfig conf, RemoteCmd remoteCmd, const(int)[] ignoreRsyncErrorCodes) {
31         this.conf = conf;
32         this.remoteCmd_ = remoteCmd;
33         this.ignoreRsyncErrorCodes = ignoreRsyncErrorCodes;
34     }
35 
36     override void remoteCmd(const RemoteHost host, const RemoteSubCmd cmd_, const string path) {
37         import std.path : buildPath;
38 
39         auto cmd = remoteCmd_.match!((SshRemoteCmd a) {
40             return a.toCmd(cmd_, host.addr, buildPath(host.path, path));
41         });
42 
43         // TODO: throw exception on failure?
44         spawnProcessLog(cmd).wait;
45     }
46 
47     override Layout update(Layout layout) {
48         import dsnapshot.layout_utils;
49 
50         return fillLayout(layout, conf.flow, remoteCmd_);
51     }
52 
53     override void publishSnapshot(const string newSnapshot) {
54         static import dsnapshot.cmdgroup.remote;
55 
56         conf.flow.match!((None a) {}, (FlowLocal a) {
57             dsnapshot.cmdgroup.remote.publishSnapshot((a.dst.value.Path ~ newSnapshot).toString);
58         }, (FlowRsyncToLocal a) {
59             dsnapshot.cmdgroup.remote.publishSnapshot((a.dst.value.Path ~ newSnapshot).toString);
60         }, (FlowLocalToRsync a) {
61             this.remoteCmd(RemoteHost(a.dst.addr, a.dst.path),
62                 RemoteSubCmd.publishSnapshot, newSnapshot);
63         });
64     }
65 
66     override void removeDiscarded(const Layout layout) {
67         void local(const LocalAddr local) @safe {
68             import std.file : rmdirRecurse, exists, isDir;
69 
70             foreach (const old; layout.discarded
71                     .map!(a => (local.value.Path ~ a.name.value).toString)
72                     .filter!(a => exists(a) && a.isDir)) {
73                 logger.info("Removing old snapshot ", old);
74                 try {
75                     () @trusted { rmdirRecurse(old); }();
76                 } catch (Exception e) {
77                     logger.warning(e.msg);
78                 }
79             }
80         }
81 
82         void remote(const RemoteHost host) @safe {
83             foreach (const name; layout.discarded.map!(a => a.name)) {
84                 logger.info("Removing old snapshot ", name.value);
85                 this.remoteCmd(host, RemoteSubCmd.rmdirRecurse, name.value);
86             }
87         }
88 
89         conf.flow.match!((None a) {}, (FlowLocal a) { local(a.dst); }, (FlowRsyncToLocal a) {
90             local(a.dst);
91         }, (FlowLocalToRsync a) { remote(RemoteHost(a.dst.addr, a.dst.path)); });
92     }
93 
94     override void sync(const Layout layout, const SnapshotConfig snapshot,
95             const string nameOfNewSnapshot) {
96         import std.algorithm : canFind;
97         import std.array : replace, array, empty;
98         import std.conv : to;
99         import std.file : remove, exists, mkdirRecurse;
100         import std.format : format;
101         import std.path : buildPath, setExtension;
102         import std.process : spawnShell;
103         import std.stdio : stdin, File;
104         import dsnapshot.console;
105 
106         static void setupLocalDest(const Path p) @safe {
107             auto dst = p ~ snapshotData;
108             if (!exists(dst.toString))
109                 mkdirRecurse(dst.toString);
110         }
111 
112         static void setupRemoteDest(const RemoteCmd remoteCmd,
113                 const RemoteHost addr, const string newSnapshot) @safe {
114             string[] cmd = remoteCmd.match!((const SshRemoteCmd a) {
115                 return a.toCmd(RemoteSubCmd.mkdirRecurse, addr.addr,
116                     buildPath(addr.path, newSnapshot, snapshotData));
117             });
118             if (!cmd.empty) {
119                 spawnProcessLog(cmd).wait;
120             }
121         }
122 
123         static int executeHooks(const string msg, const string[] hooks, const string[string] env) @safe {
124             foreach (s; hooks) {
125                 logger.info(msg, ": ", s);
126                 if (spawnShell(s, env).wait != 0)
127                     return 1;
128             }
129             return 0;
130         }
131 
132         string src, dst;
133 
134         string[] buildOpts() @safe {
135             string[] opts = [conf.cmdRsync];
136             opts ~= conf.backupArgs.dup;
137 
138             const latest = layout.firstFullBucket;
139 
140             if (!conf.rsh.empty)
141                 opts ~= ["-e", conf.rsh];
142 
143             if (isInteractiveShell)
144                 opts ~= conf.progress;
145 
146             if (conf.oneFs && !latest.isNull)
147                 opts ~= ["-x"];
148 
149             if (conf.useLinkDest && !latest.isNull) {
150                 conf.flow.match!((None a) {}, (FlowLocal a) {
151                     opts ~= [
152                         "--link-dest",
153                         (a.dst.value.Path ~ latest.get.name.value ~ snapshotData).toString
154                     ];
155                 }, (FlowRsyncToLocal a) {
156                     opts ~= [
157                         "--link-dest",
158                         (a.dst.value.Path ~ latest.get.name.value ~ snapshotData).toString
159                     ];
160                 }, (FlowLocalToRsync a) {
161                     // from the rsync documentation:
162                     // If DIR is a relative path, it is relative to the destination directory.
163                     opts ~= [
164                         "--link-dest",
165                         buildPath("..", "..", latest.get.name.value, snapshotData)
166                     ];
167                 });
168             }
169 
170             foreach (a; conf.exclude)
171                 opts ~= ["--exclude", a];
172 
173             if (conf.useFakeRoot) {
174                 conf.flow.match!((None a) {}, (FlowLocal a) {
175                     opts = conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
176                         (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotFakerootEnv).toString)).array
177                         ~ opts;
178                 }, (FlowRsyncToLocal a) {
179                     opts = conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
180                         (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotFakerootEnv).toString)).array
181                         ~ opts;
182                 }, (FlowLocalToRsync a) {
183                     opts ~= conf.rsyncFakerootArgs;
184                     opts ~= format!"%-(%s %) %s"(conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
185                         buildPath(a.dst.path, nameOfNewSnapshot, snapshotFakerootEnv))),
186                         conf.cmdRsync);
187                 });
188             }
189 
190             conf.flow.match!((None a) {}, (FlowLocal a) {
191                 src = fixRemteHostForRsync(a.src.value);
192                 dst = (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotData).toString;
193                 opts ~= [src, dst];
194             }, (FlowRsyncToLocal a) {
195                 src = fixRemteHostForRsync(makeRsyncAddr(a.src.addr, a.src.path));
196                 dst = (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotData).toString;
197                 opts ~= [src, dst];
198             }, (FlowLocalToRsync a) {
199                 src = fixRemteHostForRsync(a.src.value);
200                 dst = makeRsyncAddr(a.dst.addr, buildPath(a.dst.path,
201                     nameOfNewSnapshot, snapshotData));
202                 opts ~= [src, dst];
203             });
204 
205             return opts;
206         }
207 
208         auto opts = buildOpts();
209 
210         if (src.empty || dst.empty) {
211             logger.info("source or destination is not configured. Nothing to do.");
212             return;
213         }
214 
215         // Configure local destination
216         conf.flow.match!((None a) {}, (FlowLocal a) => setupLocalDest(a.dst.value.Path ~ nameOfNewSnapshot),
217                 (FlowRsyncToLocal a) => setupLocalDest(a.dst.value.Path ~ nameOfNewSnapshot),
218                 (FlowLocalToRsync a) => setupRemoteDest(remoteCmd_, a.dst, nameOfNewSnapshot));
219 
220         logger.infof("Synchronizing '%s' to '%s'", src, dst);
221 
222         string[string] hookEnv = ["DSNAPSHOT_SRC" : src, "DSNAPSHOT_DST" : dst];
223 
224         if (executeHooks("pre_exec", snapshot.hooks.preExec, hookEnv) != 0)
225             throw new SnapshotException(SnapshotException.PreExecFailed.init.SnapshotError);
226 
227         auto syncPid = spawnProcessLog(opts);
228 
229         if (conf.lowPrio) {
230             try {
231                 logger.infof("Changing IO and CPU priority to low (pid %s)", syncPid.processID);
232                 executeLog([
233                         "ionice", "-c", "3", "-p", syncPid.processID.to!string
234                         ]);
235                 executeLog(["renice", "+12", "-p", syncPid.processID.to!string]);
236             } catch (Exception e) {
237                 logger.info(e.msg);
238             }
239         }
240 
241         auto syncPidExit = syncPid.wait;
242         logger.trace("rsync exit code: ", syncPidExit);
243         if (syncPidExit != 0 && !canFind(ignoreRsyncErrorCodes, syncPidExit))
244             throw new SnapshotException(SnapshotException.SyncFailed(src, dst).SnapshotError);
245 
246         if (executeHooks("post_exec", snapshot.hooks.postExec, hookEnv) != 0)
247             throw new SnapshotException(SnapshotException.PostExecFailed.init.SnapshotError);
248     }
249 
250     override void restore(const Layout layout, const SnapshotConfig snapshot,
251             const SysTime time, const string restoreTo) {
252         import std.array : empty;
253         import std.file : exists, mkdirRecurse;
254         import std.path : buildPath;
255         import dsnapshot.console : isInteractiveShell;
256 
257         const bestFitSnapshot = layout.bestFitBucket(time);
258         if (bestFitSnapshot.isNull) {
259             logger.error("Unable to find a snapshot to restore for the time ", time);
260             throw new SnapshotException(SnapshotException.RestoreFailed(null,
261                     restoreTo).SnapshotError);
262         }
263 
264         string src;
265 
266         // TODO: this code is similare to the one in cmdgroup.backup. Consider how
267         // it can be deduplicated. Note, similare not the same.
268         string[] buildOpts() {
269             string[] opts = [conf.cmdRsync];
270             opts ~= conf.restoreArgs.dup;
271 
272             if (!conf.rsh.empty)
273                 opts ~= ["-e", conf.rsh];
274 
275             if (isInteractiveShell)
276                 opts ~= conf.progress;
277 
278             foreach (a; conf.exclude)
279                 opts ~= ["--exclude", a];
280 
281             conf.flow.match!((None a) {}, (FlowLocal a) {
282                 src = fixRemteHostForRsync((a.dst.value.Path ~ bestFitSnapshot.name.value ~ snapshotData)
283                     .toString);
284             }, (FlowRsyncToLocal a) {
285                 src = fixRemteHostForRsync((a.dst.value.Path ~ bestFitSnapshot.name.value ~ snapshotData)
286                     .toString);
287             }, (FlowLocalToRsync a) {
288                 src = makeRsyncAddr(a.dst.addr, fixRemteHostForRsync(buildPath(a.dst.path,
289                     bestFitSnapshot.name.value, snapshotData)));
290             });
291 
292             opts ~= src;
293             // dst is always on the local machine as specified by the user
294             opts ~= restoreTo;
295             return opts;
296         }
297 
298         const opts = buildOpts();
299 
300         if (!exists(restoreTo))
301             mkdirRecurse(restoreTo);
302 
303         logger.infof("Restoring %s to %s", bestFitSnapshot.name.value, restoreTo);
304 
305         if (spawnProcessLog(opts).wait != 0)
306             throw new SnapshotException(SnapshotException.RestoreFailed(src,
307                     restoreTo).SnapshotError);
308 
309         if (conf.useFakeRoot) {
310             conf.flow.match!((None a) {}, (FlowLocal a) => fakerootLocalRestore(
311                     a.dst.value.Path ~ bestFitSnapshot.name.value, restoreTo), (FlowRsyncToLocal a) {
312                 logger.warning("Restoring permissions to a remote is not supported (yet)");
313             }, (FlowLocalToRsync a) => fakerootRemoteRestore(snapshot.remoteCmd,
314                     a.dst, bestFitSnapshot.name, restoreTo));
315         }
316     }
317 
318     Flow flow() {
319         return conf.flow;
320     }
321 }
322 
323 private:
324 
325 void fakerootLocalRestore(const Path root, const string restoreTo) {
326     import dsnapshot.stats;
327 
328     auto fkdb = fromFakerootEnv(root ~ snapshotFakerootEnv);
329     auto pstats = fromFakeroot(fkdb, root.toString, (root ~ snapshotData).toString);
330     restorePermissions(pstats, Path(restoreTo));
331 }
332 
333 void fakerootRemoteRestore(const RemoteCmd cmd_, const RemoteHost addr,
334         const Name name, const string restoreTo) {
335     import std.array : appender, empty;
336     import std.path : buildPath;
337     import std.string : lineSplitter, strip;
338     import dsnapshot.stats;
339 
340     auto cmd = cmd_.match!((const SshRemoteCmd a) {
341         return a.toCmd(RemoteSubCmd.fakerootStats, addr.addr, buildPath(addr.path, name.value));
342     });
343 
344     auto res = executeLog(cmd);
345 
346     if (res.status != 0) {
347         logger.errorf("Unable to restore permissions to %s from %s", restoreTo,
348                 snapshotFakerootEnv);
349         logger.info(res.output);
350         return;
351     }
352 
353     auto app = appender!(PathStat[])();
354     foreach (const l; res.output
355             .lineSplitter
356             .map!(a => a.strip)
357             .filter!(a => !a.empty)) {
358         try {
359             app.put(fromPathStat(l));
360         } catch (Exception e) {
361             logger.warning("Error when parsing ", l);
362             logger.info(e.msg);
363         }
364     }
365 
366     restorePermissions(app.data, Path(restoreTo));
367 }
368 
369 void restorePermissions(const from.dsnapshot.stats.PathStat[] pstats, const Path root) @trusted {
370     import core.sys.posix.sys.stat : S_IFMT, stat_t, stat, lstat, chmod;
371     import core.sys.posix.unistd : chown, lchown;
372     import std.file : isFile, isDir, isSymlink;
373     import std.string : toStringz;
374 
375     foreach (const f; pstats) {
376         const curr = (root ~ f.path).toString;
377         const currz = curr.toStringz;
378 
379         if (curr.isFile || curr.isDir) {
380             stat_t st = void;
381             stat(currz, &st);
382             if (st.st_mode != f.mode) {
383                 // only set the permissions thus masking out other bits.
384                 chmod(currz, cast(uint) f.mode & ~S_IFMT);
385             }
386             if (st.st_uid != f.uid || st.st_gid != f.gid)
387                 chown(currz, cast(uint) f.uid, cast(uint) f.gid);
388         } else if (curr.isSymlink) {
389             stat_t st = void;
390             lstat(currz, &st);
391             if (st.st_uid != f.uid || st.st_gid != f.gid)
392                 lchown(currz, cast(uint) f.uid, cast(uint) f.gid);
393         }
394     }
395 }