dsnapshot.backend.rsync source code

1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.backend.rsync;
7 
8 import logger = std.experimental.logger;
9 import std.algorithm : map, filter;
10 import std.datetime : SysTime;
11 
12 import dsnapshot.backend.crypt;
13 import dsnapshot.backend.rsync;
14 import dsnapshot.backend;
15 import dsnapshot.config;
16 import dsnapshot.exception;
17 import dsnapshot.from;
18 import dsnapshot.layout : Layout;
19 import dsnapshot.process;
20 import dsnapshot.types;
21 
22 @safe:
23 
24 final class RsyncBackend : SyncBackend {
25     RsyncConfig conf;
26     RemoteCmd remoteCmd_;
27     /// Error codes ignored when Synchronizing.
28     const(int)[] ignoreRsyncErrorCodes;
29 
30     this(RsyncConfig conf, RemoteCmd remoteCmd, const(int)[] ignoreRsyncErrorCodes) {
31         this.conf = conf;
32         this.remoteCmd_ = remoteCmd;
33         this.ignoreRsyncErrorCodes = ignoreRsyncErrorCodes;
34     }
35 
36     override void remoteCmd(const RemoteHost host, const RemoteSubCmd cmd_, const string path) {
37         import std.path : buildPath;
38 
39         auto cmd = remoteCmd_.match!((SshRemoteCmd a) {
40             return a.toCmd(cmd_, host.addr, buildPath(host.path, path));
41         });
42 
43         // TODO: throw exception on failure?
44         spawnProcessLog(cmd).wait;
45     }
46 
47     override Layout update(Layout layout) {
48         import dsnapshot.layout_utils;
49         import std.datetime : Clock;
50 
51         return fillLayout(Layout(Clock.currTime, layout.conf), conf.flow, remoteCmd_);
52     }
53 
54     override void publishSnapshot(const string newSnapshot) {
55         static import dsnapshot.cmdgroup.remote;
56 
57         conf.flow.match!((None a) {}, (FlowLocal a) {
58             dsnapshot.cmdgroup.remote.publishSnapshot((a.dst.value.Path ~ newSnapshot).toString);
59         }, (FlowRsyncToLocal a) {
60             dsnapshot.cmdgroup.remote.publishSnapshot((a.dst.value.Path ~ newSnapshot).toString);
61         }, (FlowLocalToRsync a) {
62             this.remoteCmd(RemoteHost(a.dst.addr, a.dst.path),
63                 RemoteSubCmd.publishSnapshot, newSnapshot);
64         });
65     }
66 
67     override void removeDiscarded(const Layout layout) {
68         void local(const LocalAddr local) @safe {
69             import std.file : rmdirRecurse, exists, isDir;
70 
71             foreach (const old; layout.discarded
72                     .map!(a => (local.value.Path ~ a.name.value).toString)
73                     .filter!(a => exists(a) && a.isDir)) {
74                 logger.info("Removing old snapshot ", old);
75                 try {
76                     () @trusted { rmdirRecurse(old); }();
77                 } catch (Exception e) {
78                     logger.warning(e.msg);
79                 }
80             }
81         }
82 
83         void remote(const RemoteHost host) @safe {
84             foreach (const name; layout.discarded.map!(a => a.name)) {
85                 logger.info("Removing old snapshot ", name.value);
86                 this.remoteCmd(host, RemoteSubCmd.rmdirRecurse, name.value);
87             }
88         }
89 
90         conf.flow.match!((None a) {}, (FlowLocal a) { local(a.dst); }, (FlowRsyncToLocal a) {
91             local(a.dst);
92         }, (FlowLocalToRsync a) { remote(RemoteHost(a.dst.addr, a.dst.path)); });
93     }
94 
95     override void sync(const Layout layout, const SnapshotConfig snapshot,
96             const string nameOfNewSnapshot) {
97         import std.algorithm : canFind;
98         import std.array : replace, array, empty;
99         import std.conv : to;
100         import std.file : remove, exists, mkdirRecurse;
101         import std.format : format;
102         import std.path : buildPath, setExtension, dirName, baseName;
103         import std.process : spawnShell, executeShell;
104         import std.stdio : stdin, File, write;
105         import dsnapshot.console;
106 
107         static void setupLocalDest(const Path p) @safe {
108             auto dst = p ~ snapshotData;
109             if (!exists(dst.toString))
110                 mkdirRecurse(dst.toString);
111         }
112 
113         static void setupRemoteDest(const RemoteCmd remoteCmd,
114                 const RemoteHost addr, const string newSnapshot) @safe {
115             string[] cmd = remoteCmd.match!((const SshRemoteCmd a) {
116                 return a.toCmd(RemoteSubCmd.mkdirRecurse, addr.addr,
117                     buildPath(addr.path, newSnapshot, snapshotData));
118             });
119             if (!cmd.empty) {
120                 spawnProcessLog(cmd).wait;
121             }
122         }
123 
124         static int executeHooks(const string msg, const string[] hooks, const string[string] env) @safe {
125             foreach (s; hooks) {
126                 logger.info(msg, ": ", s);
127                 if (isInteractiveShell) {
128                     if (spawnShell(s, env).wait != 0)
129                         return 1;
130                 } else {
131                     auto res = executeShell(s, env);
132                     write(res.output);
133                     if (res.status != 0)
134                         return 1;
135                 }
136             }
137             return 0;
138         }
139 
140         string src, dst, latest;
141 
142         string[] buildOpts() @safe {
143             string[] opts = [conf.cmdRsync];
144             opts ~= conf.backupArgs.dup;
145             const latest_ = layout.firstFullBucket;
146 
147             conf.flow.match!((None a) {}, (FlowLocal a) {
148                 src = fixRemteHostForRsync(a.src.value);
149                 dst = (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotData).toString;
150                 if (!latest_.isNull)
151                     latest = (a.dst.value.Path ~ latest_.get.name.value).toString;
152             }, (FlowRsyncToLocal a) {
153                 src = fixRemteHostForRsync(makeRsyncAddr(a.src.addr, a.src.path));
154                 dst = (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotData).toString;
155                 if (!latest_.isNull)
156                     latest = (a.dst.value.Path ~ latest_.get.name.value).toString;
157             }, (FlowLocalToRsync a) {
158                 src = fixRemteHostForRsync(a.src.value);
159                 dst = makeRsyncAddr(a.dst.addr, buildPath(a.dst.path,
160                     nameOfNewSnapshot, snapshotData));
161                 if (!latest_.isNull)
162                     latest = buildPath(a.dst.path, latest_.get.name.value);
163             });
164 
165             if (!conf.rsh.empty)
166                 opts ~= ["-e", conf.rsh];
167 
168             if (isInteractiveShell)
169                 opts ~= conf.progress;
170 
171             if (conf.oneFs && !latest_.isNull)
172                 opts ~= ["-x"];
173 
174             if (conf.useLinkDest && !latest_.isNull) {
175                 conf.flow.match!((None a) {}, (FlowLocal a) {
176                     opts ~= ["--link-dest", buildPath(latest, snapshotData)];
177                 }, (FlowRsyncToLocal a) {
178                     opts ~= ["--link-dest", buildPath(latest, snapshotData)];
179                 }, (FlowLocalToRsync a) {
180                     // from the rsync documentation:
181                     // If DIR is a relative path, it is relative to the destination directory.
182                     opts ~= [
183                         "--link-dest",
184                         buildPath("..", "..", latest.baseName, snapshotData)
185                     ];
186                 });
187             }
188 
189             foreach (a; conf.exclude)
190                 opts ~= ["--exclude", a];
191 
192             if (conf.useFakeRoot) {
193                 conf.flow.match!((None a) {}, (FlowLocal a) {
194                     opts = conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
195                         (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotFakerootEnv).toString)).array
196                         ~ opts;
197                 }, (FlowRsyncToLocal a) {
198                     opts = conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
199                         (a.dst.value.Path ~ nameOfNewSnapshot ~ snapshotFakerootEnv).toString)).array
200                         ~ opts;
201                 }, (FlowLocalToRsync a) {
202                     opts ~= conf.rsyncFakerootArgs;
203                     opts ~= format!"%-(%s %) %s"(conf.fakerootArgs.map!(b => b.replace(snapshotFakerootSaveEnvId,
204                         buildPath(a.dst.path, nameOfNewSnapshot, snapshotFakerootEnv))),
205                         conf.cmdRsync);
206                 });
207             }
208 
209             opts ~= [src, dst];
210 
211             return opts;
212         }
213 
214         auto opts = buildOpts();
215 
216         if (src.empty || dst.empty) {
217             logger.info("source or destination is not configured. Nothing to do.");
218             return;
219         }
220 
221         // Configure local destination
222         conf.flow.match!((None a) {}, (FlowLocal a) => setupLocalDest(a.dst.value.Path ~ nameOfNewSnapshot),
223                 (FlowRsyncToLocal a) => setupLocalDest(a.dst.value.Path ~ nameOfNewSnapshot),
224                 (FlowLocalToRsync a) => setupRemoteDest(remoteCmd_, a.dst, nameOfNewSnapshot));
225 
226         logger.infof("Synchronizing '%s' to '%s'", src, dst);
227 
228         string[string] hookEnv = [
229             "DSNAPSHOT_SRC" : src, "DSNAPSHOT_DST" : dst.dirName,
230             "DSNAPSHOT_DATA_DST" : dst, "DSNAPSHOT_LATEST" : latest,
231             "DSNAPSHOT_DATA_LATEST" : buildPath(latest, snapshotData),
232         ];
233 
234         if (executeHooks("pre_exec", snapshot.hooks.preExec, hookEnv) != 0)
235             throw new SnapshotException(SnapshotException.PreExecFailed.init.SnapshotError);
236 
237         auto syncPid = spawnProcessLog(opts);
238 
239         if (conf.lowPrio) {
240             try {
241                 logger.infof("Changing IO and CPU priority to low (pid %s)", syncPid.processID);
242                 executeLog([
243                         "ionice", "-c", "3", "-p", syncPid.processID.to!string
244                         ]);
245                 executeLog(["renice", "+12", "-p", syncPid.processID.to!string]);
246             } catch (Exception e) {
247                 logger.info(e.msg);
248             }
249         }
250 
251         auto syncPidExit = syncPid.wait;
252         logger.trace("rsync exit code: ", syncPidExit);
253         if (syncPidExit != 0 && !canFind(ignoreRsyncErrorCodes, syncPidExit))
254             throw new SnapshotException(SnapshotException.SyncFailed(src, dst).SnapshotError);
255 
256         if (executeHooks("post_exec", snapshot.hooks.postExec, hookEnv) != 0)
257             throw new SnapshotException(SnapshotException.PostExecFailed.init.SnapshotError);
258     }
259 
260     override void restore(const Layout layout, const SnapshotConfig snapshot,
261             const SysTime time, const string restoreTo) {
262         import std.array : empty;
263         import std.file : exists, mkdirRecurse;
264         import std.path : buildPath;
265         import dsnapshot.console : isInteractiveShell;
266 
267         const bestFitSnapshot = layout.bestFitBucket(time);
268         if (bestFitSnapshot.isNull) {
269             logger.error("Unable to find a snapshot to restore for the time ", time);
270             throw new SnapshotException(SnapshotException.RestoreFailed(null,
271                     restoreTo).SnapshotError);
272         }
273 
274         string src;
275 
276         // TODO: this code is similare to the one in cmdgroup.backup. Consider how
277         // it can be deduplicated. Note, similare not the same.
278         string[] buildOpts() {
279             string[] opts = [conf.cmdRsync];
280             opts ~= conf.restoreArgs.dup;
281 
282             if (!conf.rsh.empty)
283                 opts ~= ["-e", conf.rsh];
284 
285             if (isInteractiveShell)
286                 opts ~= conf.progress;
287 
288             foreach (a; conf.exclude)
289                 opts ~= ["--exclude", a];
290 
291             conf.flow.match!((None a) {}, (FlowLocal a) {
292                 src = fixRemteHostForRsync((a.dst.value.Path ~ bestFitSnapshot.name.value ~ snapshotData)
293                     .toString);
294             }, (FlowRsyncToLocal a) {
295                 src = fixRemteHostForRsync((a.dst.value.Path ~ bestFitSnapshot.name.value ~ snapshotData)
296                     .toString);
297             }, (FlowLocalToRsync a) {
298                 src = makeRsyncAddr(a.dst.addr, fixRemteHostForRsync(buildPath(a.dst.path,
299                     bestFitSnapshot.name.value, snapshotData)));
300             });
301 
302             opts ~= src;
303             // dst is always on the local machine as specified by the user
304             opts ~= restoreTo;
305             return opts;
306         }
307 
308         const opts = buildOpts();
309 
310         if (!exists(restoreTo))
311             mkdirRecurse(restoreTo);
312 
313         logger.infof("Restoring %s to %s", bestFitSnapshot.name.value, restoreTo);
314 
315         if (spawnProcessLog(opts).wait != 0)
316             throw new SnapshotException(SnapshotException.RestoreFailed(src,
317                     restoreTo).SnapshotError);
318 
319         if (conf.useFakeRoot) {
320             conf.flow.match!((None a) {}, (FlowLocal a) => fakerootLocalRestore(
321                     a.dst.value.Path ~ bestFitSnapshot.name.value, restoreTo), (FlowRsyncToLocal a) {
322                 logger.warning("Restoring permissions to a remote is not supported (yet)");
323             }, (FlowLocalToRsync a) => fakerootRemoteRestore(snapshot.remoteCmd,
324                     a.dst, bestFitSnapshot.name, restoreTo));
325         }
326     }
327 
328     Flow flow() {
329         return conf.flow;
330     }
331 }
332 
333 private:
334 
335 void fakerootLocalRestore(const Path root, const string restoreTo) {
336     import dsnapshot.stats;
337 
338     auto fkdb = fromFakerootEnv(root ~ snapshotFakerootEnv);
339     auto pstats = fromFakeroot(fkdb, root.toString, (root ~ snapshotData).toString);
340     restorePermissions(pstats, Path(restoreTo));
341 }
342 
343 void fakerootRemoteRestore(const RemoteCmd cmd_, const RemoteHost addr,
344         const Name name, const string restoreTo) {
345     import std.array : appender, empty;
346     import std.path : buildPath;
347     import std.string : lineSplitter, strip;
348     import dsnapshot.stats;
349 
350     auto cmd = cmd_.match!((const SshRemoteCmd a) {
351         return a.toCmd(RemoteSubCmd.fakerootStats, addr.addr, buildPath(addr.path, name.value));
352     });
353 
354     auto res = executeLog(cmd);
355 
356     if (res.status != 0) {
357         logger.errorf("Unable to restore permissions to %s from %s", restoreTo,
358                 snapshotFakerootEnv);
359         logger.info(res.output);
360         return;
361     }
362 
363     auto app = appender!(PathStat[])();
364     foreach (const l; res.output
365             .lineSplitter
366             .map!(a => a.strip)
367             .filter!(a => !a.empty)) {
368         try {
369             app.put(fromPathStat(l));
370         } catch (Exception e) {
371             logger.warning("Error when parsing ", l);
372             logger.info(e.msg);
373         }
374     }
375 
376     restorePermissions(app.data, Path(restoreTo));
377 }
378 
379 void restorePermissions(const from.dsnapshot.stats.PathStat[] pstats, const Path root) @trusted {
380     import core.sys.posix.sys.stat : S_IFMT, stat_t, stat, lstat, chmod;
381     import core.sys.posix.unistd : chown, lchown;
382     import std.file : isFile, isDir, isSymlink;
383     import std.string : toStringz;
384 
385     foreach (const f; pstats) {
386         const curr = (root ~ f.path).toString;
387         const currz = curr.toStringz;
388 
389         if (curr.isFile || curr.isDir) {
390             stat_t st = void;
391             stat(currz, &st);
392             if (st.st_mode != f.mode) {
393                 // only set the permissions thus masking out other bits.
394                 chmod(currz, cast(uint) f.mode & ~S_IFMT);
395             }
396             if (st.st_uid != f.uid || st.st_gid != f.gid)
397                 chown(currz, cast(uint) f.uid, cast(uint) f.gid);
398         } else if (curr.isSymlink) {
399             stat_t st = void;
400             lstat(currz, &st);
401             if (st.st_uid != f.uid || st.st_gid != f.gid)
402                 lchown(currz, cast(uint) f.uid, cast(uint) f.gid);
403         }
404     }
405 }