app source code

1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module app;
7 
8 import logger = std.experimental.logger;
9 import std.algorithm : remove, map, filter;
10 import std.array : array, empty;
11 import std.exception : collectException;
12 import std.path : baseName, expandTilde;
13 import std.stdio : writeln;
14 
15 import colorlog;
16 
17 import dsnapshot.config;
18 
19 int main(string[] args) {
20     import dsnapshot.cmdgroup.admin;
21     import dsnapshot.cmdgroup.backup;
22     import dsnapshot.cmdgroup.remote;
23     import dsnapshot.cmdgroup.restore;
24     import dsnapshot.cmdgroup.watch;
25 
26     confLogger(VerboseMode.info);
27 
28     auto conf = parseUserArgs(args);
29 
30     confLogger(conf.global.verbosity);
31 
32     logger.trace(conf.global);
33     static foreach (T; Config.Type.AllowedTypes) {
34         if (auto a = conf.data.peek!T)
35             logger.tracef("%s", *a);
36     }
37 
38     if (conf.global.help) {
39         return cmdHelp(conf);
40     }
41 
42     import std.variant : visit;
43 
44     // dfmt off
45     return conf.data.visit!(
46       (Config.Help a) => cmdHelp(conf),
47       (Config.Backup a) {
48           loadConfig(conf);
49           return cmdBackup(conf.global, a, conf.snapshots);
50       },
51       (Config.Remotecmd a) => cmdRemote(a),
52       (Config.Restore a) {
53           loadConfig(conf);
54           return cmdRestore(conf.snapshots, a);
55       },
56       (Config.Verifyconfig a) {
57           loadConfig(conf);
58           logger.info("Done");
59           return 0;
60       },
61       (Config.Admin a) {
62           loadConfig(conf);
63           return cmdAdmin(conf.snapshots, a);
64       },
65       (Config.Watch a) {
66           loadConfig(conf);
67           return cli(conf.global, a, conf.snapshots);
68       }
69     );
70     // dfmt on
71 }
72 
73 @safe:
74 private:
75 
76 int cmdHelp(Config conf) @trusted {
77     conf.printHelp;
78     return 0;
79 }
80 
81 Config parseUserArgs(string[] args) @trusted {
82     import std.format : format;
83     import std.string : toLower;
84     import std.traits : EnumMembers;
85     static import std.getopt;
86 
87     Config conf;
88     conf.data = Config.Help.init;
89     conf.global.progName = args[0].baseName;
90     conf.global.confFile = ".dsnapshot.toml";
91 
92     string group;
93     if (args.length > 1) {
94         group = args[1];
95         args = args.remove(1);
96     }
97 
98     try {
99         void globalParse() {
100             Config.Help data;
101             scope (success)
102                 conf.data = data;
103 
104             string confFile;
105             // dfmt off
106             data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough,
107                 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity,
108                 "c|config", "Config file to read", &confFile,
109                 );
110             // dfmt on
111             conf.global.help = data.helpInfo.helpWanted;
112             args ~= (conf.global.help ? "-h" : null);
113 
114             if (!confFile.empty)
115                 conf.global.confFile = confFile.Path;
116         }
117 
118         void backupParse() {
119             Config.Backup data;
120             scope (success)
121                 conf.data = data;
122 
123             string margin;
124             // dfmt off
125             data.helpInfo = std.getopt.getopt(args,
126                 "force", "Force a backup to be taken even though it isn't time for it", &data.forceBackup,
127                 "ignore-rsync-error-code", "Ignore rsync error code", &data.ignoreRsyncErrorCodes,
128                 "margin", "Add a margin when checking if a new snapshot should be taken", &margin,
129                 "resume", "If an interrupted backup should be resumed", &data.resume,
130                 "s|snapshot", "The name of the snapshot to backup (default: all)", &data.name.value,
131                 );
132             // dfmt on
133             data.newSnapshotMargin = parseDuration(margin);
134         }
135 
136         void remotecmdParse() {
137             Config.Remotecmd data;
138             scope (success)
139                 conf.data = data;
140 
141             // dfmt off
142             data.helpInfo = std.getopt.getopt(args,
143                 "cmd", format("Command to execute (%-(%s, %))", [EnumMembers!(RemoteSubCmd)]), &data.cmd,
144                 "path", "Path argument for the command", &data.path,
145                 );
146             // dfmt on
147         }
148 
149         void restoreParse() {
150             import std.datetime : SysTime, UTC, Clock;
151 
152             Config.Restore data;
153             scope (success)
154                 conf.data = data;
155 
156             string time;
157             // dfmt off
158             data.helpInfo = std.getopt.getopt(args,
159                 "delete", "Delete files from dst if they are removed in the snapshot", &data.deleteFromTo,
160                 "dst", "Where to restore the snapshot", &data.restoreTo,
161                 "s|snapshot", "Name of the snapshot to calculate the disk usage for", &data.name.value,
162                 "time", "Pick the snapshot that is closest to this time (default: now). Add a trailing Z if UTC", &time,
163                 );
164             // dfmt on
165 
166             try {
167                 if (time.length == 0)
168                     data.time = Clock.currTime;
169                 else
170                     data.time = SysTime.fromISOExtString(time);
171             } catch (Exception e) {
172                 logger.error(e.msg);
173                 throw new Exception("Example of UTC time: 2019-07-18T10:49:29.5765454Z");
174             }
175         }
176 
177         void verifyconfigParse() {
178             conf.data = Config.Verifyconfig.init;
179         }
180 
181         void adminParse() {
182             Config.Admin data;
183             scope (success)
184                 conf.data = data;
185 
186             string[] names;
187             // dfmt off
188             data.helpInfo = std.getopt.getopt(args,
189                 "s|snapshot", "Snapshot name (default: all)", &names,
190                 "cmd", format!"What to do. Available: %-(%s, %) (default: list)"([EnumMembers!(Config.Admin.Cmd)]), &data.cmd,
191                 );
192             // dfmt on
193 
194             data.names = names.map!(a => Name(a)).array;
195         }
196 
197         void watchParse() {
198             Config.Watch data;
199             scope (success)
200                 conf.data = data;
201 
202             // dfmt off
203             data.helpInfo = std.getopt.getopt(args,
204                 "max-nr", "Exit after this number of snapshots has been created", &data.maxSnapshots,
205                 std.getopt.config.required, "s|snapshot", "The name of the snapshot to backup", &data.name.value,
206                 );
207             // dfmt on
208         }
209 
210         alias ParseFn = void delegate();
211         ParseFn[string] parsers;
212 
213         static foreach (T; Config.Type.AllowedTypes) {
214             static if (!is(T == Config.Help))
215                 mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower));
216         }
217 
218         globalParse;
219 
220         if (auto p = group in parsers) {
221             (*p)();
222         }
223     } catch (std.getopt.GetOptException e) {
224         // unknown option
225         conf.global.help = true;
226         logger.error(e.msg);
227     } catch (Exception e) {
228         conf.global.help = true;
229         logger.error(e.msg);
230     }
231 
232     return conf;
233 }
234 
235 void loadConfig(ref Config conf) @trusted {
236     import std.conv : to;
237     import std.file : exists, readText, isFile;
238     import std.path : dirName, buildPath;
239     import toml;
240 
241     scope (exit)
242         logger.trace(conf);
243 
244     if (!exists(conf.global.confFile.toString) || !conf.global.confFile.toString.isFile) {
245         logger.errorf("Configuration %s do not exist", conf.global.confFile);
246         return;
247     }
248 
249     static auto tryLoading(string configFile) {
250         auto txt = readText(configFile);
251         auto doc = parseTOML(txt);
252         return doc;
253     }
254 
255     TOMLDocument doc;
256     try {
257         doc = tryLoading(conf.global.confFile.toString);
258     } catch (Exception e) {
259         logger.warning("Unable to read the configuration from ", conf.global.confFile);
260         logger.warning(e.msg);
261         return;
262     }
263 
264     alias Fn = void delegate(ref Config c, ref TOMLValue v);
265     Fn[string] tables;
266 
267     tables["snapshot"] = (ref Config c, ref TOMLValue snapshots) {
268         foreach (name, data; snapshots) {
269             SnapshotConfig s;
270             s.name = name;
271             s.layout = makeDefaultLayout;
272             foreach (k, v; data) {
273                 try {
274                     switch (k) {
275                     case "rsync":
276                         auto rsync = parseRsync(v, name);
277                         s.syncCmd = rsync;
278                         break;
279                     case "encfs":
280                         s.crypt = parseEncfs(v, name);
281                         break;
282                     case "pre_exec":
283                         s.hooks.preExec = v.array.map!(a => a.str).array;
284                         break;
285                     case "post_exec":
286                         s.hooks.postExec = v.array.map!(a => a.str).array;
287                         break;
288                     case "span":
289                         auto layout = parseLayout(v);
290                         s.layout = layout;
291                         break;
292                     case "dsnapshot":
293                         s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) {
294                             a.dsnapshot = v.str;
295                             return a;
296                         });
297                         break;
298                     case "rsh":
299                         s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) {
300                             a.rsh = v.array.map!(a => a.str).array;
301                             return a;
302                         });
303                         break;
304                     default:
305                         logger.infof("Unknown option '%s' in section 'snapshot.%s' in configuration",
306                                 k, name);
307                     }
308                 } catch (Exception e) {
309                     logger.error(e.msg).collectException;
310                 }
311             }
312             c.snapshots ~= s;
313         }
314     };
315 
316     foreach (curr; doc.byKeyValue.filter!(a => a.value.type == TOML_TYPE.TABLE)) {
317         try {
318             if (auto t = curr.key in tables) {
319                 (*t)(conf, curr.value);
320             } else {
321                 logger.infof("Unknown section '%s' in configuration", curr.key);
322             }
323         } catch (Exception e) {
324             logger.error(e.msg).collectException;
325         }
326     }
327 }
328 
329 import toml : TOMLValue;
330 
331 auto parseLayout(ref TOMLValue tv) @trusted {
332     import std.algorithm : sort;
333     import std.conv : to;
334     import std.datetime : Clock;
335     import std.range : chunks;
336     import std.string : split;
337     import std.typecons : Nullable;
338     import dsnapshot.layout : Span, LayoutConfig, Layout;
339 
340     Span[long] spans;
341 
342     static Nullable!Span parseASpan(ref TOMLValue data) {
343         typeof(return) rval;
344 
345         immutable intervalKey = "interval";
346         immutable nrKey = "nr";
347         if (intervalKey !in data || nrKey !in data) {
348             logger.warning("Missing either 'nr' or 'space' key");
349             return rval;
350         }
351 
352         auto d = parseDuration(data[intervalKey].str);
353 
354         const nr = data[nrKey].integer;
355         if (nr <= 0) {
356             logger.warning("nr must be positive");
357             return rval;
358         }
359 
360         rval = Span(cast(uint) nr, d);
361 
362         return rval;
363     }
364 
365     foreach (key, data; tv) {
366         try {
367             const idx = key.to!long;
368             auto span = parseASpan(data);
369             if (span.isNull) {
370                 logger.warning(tv);
371             } else {
372                 spans[idx] = span.get;
373             }
374         } catch (Exception e) {
375             logger.warning(e.msg);
376             logger.warning(tv);
377         }
378     }
379 
380     return Layout(Clock.currTime, LayoutConfig(spans.byKeyValue
381             .array
382             .sort!((a, b) => a.key < b.key)
383             .map!(a => a.value)
384             .array));
385 }
386 
387 auto parseRsync(ref TOMLValue tv, const string parent) @trusted {
388     import std.format : format;
389     import dsnapshot.types;
390 
391     RsyncConfig rval;
392 
393     string src, srcAddr, dst, dstAddr;
394     foreach (key, data; tv) {
395         switch (key) {
396         case "src":
397             src = data.str;
398             break;
399         case "src_addr":
400             srcAddr = data.str;
401             break;
402         case "dst":
403             dst = data.str;
404             break;
405         case "dst_addr":
406             dstAddr = data.str;
407             break;
408         case "cmd_rsync":
409             rval.cmdRsync = data.str;
410             break;
411         case "cross_fs":
412             rval.oneFs = data == true;
413             break;
414         case "fakeroot":
415             rval.useFakeRoot = data == true;
416             break;
417         case "rsync_fakeroot_args":
418             rval.rsyncFakerootArgs = data.array.map!(a => a.str).array;
419             break;
420         case "fakeroot_args":
421             rval.fakerootArgs = data.array.map!(a => a.str).array;
422             break;
423         case "link_dest":
424             rval.useLinkDest = data == true;
425             break;
426         case "low_prio":
427             rval.lowPrio = data == true;
428             break;
429         case "exclude":
430             rval.exclude = data.array.map!(a => a.str).array;
431             break;
432         case "rsync_cmd":
433             rval.cmdRsync = data.str;
434             break;
435         case "rsync_rsh":
436             rval.rsh = data.str;
437             break;
438         case "diskusage_cmd":
439             rval.cmdDiskUsage = data.array.map!(a => a.str).array;
440             break;
441         case "rsync_backup_args":
442             rval.backupArgs = data.array.map!(a => a.str).array;
443             break;
444         case "rsync_restore_args":
445             rval.restoreArgs = data.array.map!(a => a.str).array;
446             break;
447         case "progress":
448             rval.progress = data.array.map!(a => a.str).array;
449             break;
450         default:
451             logger.infof("Unknown option '%s' in section 'snapshot.%s.rsync' in configuration",
452                     key, parent);
453         }
454     }
455 
456     if (srcAddr.empty && dstAddr.empty) {
457         rval.flow = FlowLocal(LocalAddr(src.expandTilde), LocalAddr(dst));
458     } else if (!srcAddr.empty && dstAddr.empty) {
459         rval.flow = FlowRsyncToLocal(RemoteHost(srcAddr, src), LocalAddr(dst));
460     } else if (srcAddr.empty && !dstAddr.empty) {
461         rval.flow = FlowLocalToRsync(LocalAddr(src), RemoteHost(dstAddr, dst));
462     } else {
463         logger.warning("The combination of src, src_addr, dst and dst_addr is not supported. It either has to be local->local, local->remote or remote->local");
464     }
465 
466     return rval;
467 }
468 
469 auto parseEncfs(ref TOMLValue tv, const string parent) @trusted {
470     import std.format : format;
471     import dsnapshot.types;
472 
473     EncFsConfig rval;
474 
475     foreach (key, data; tv) {
476         switch (key) {
477         case "config":
478             rval.configFile = data.str;
479             break;
480         case "encrypted_path":
481             rval.encryptedPath = data.str;
482             break;
483         case "passwd":
484             rval.passwd = data.str;
485             break;
486         case "mount_cmd":
487             rval.mountCmd = data.array.map!(a => a.str).array;
488             break;
489         case "mount_fuse_opts":
490             rval.mountFuseOpts = data.array.map!(a => a.str).array;
491             break;
492         case "unmount_cmd":
493             rval.unmountCmd = data.array.map!(a => a.str).array;
494             break;
495         case "unmount_fuse_opts":
496             rval.unmountFuseOpts = data.array.map!(a => a.str).array;
497             break;
498         default:
499             logger.infof("Unknown option '%s' in section 'snapshot.%s.encfs' in configuration",
500                     key, parent);
501         }
502     }
503 
504     if (rval.encryptedPath.empty) {
505         logger.error("No 'encrypted_path' specified for ", parent);
506     }
507 
508     return rval;
509 }
510 
511 /** The default layout to use if none is specified by the user.
512  */
513 auto makeDefaultLayout() {
514     import std.datetime : Clock, dur;
515     import dsnapshot.layout;
516 
517     const base = Clock.currTime;
518     auto conf = LayoutConfig([
519             Span(6, 4.dur!"hours"), Span(6, 1.dur!"days"), Span(3, 1.dur!"weeks")
520             ]);
521     return Layout(base, conf);
522 }
523 
524 auto parseDuration(string timeSpec) {
525     import std.conv : to;
526     import std.string : split;
527     import std.datetime : Duration, dur;
528     import std.range : chunks;
529 
530     Duration d;
531     const parts = timeSpec.split;
532 
533     if (parts.length % 2 != 0) {
534         logger.warning("Invalid time specification because either the number or unit is missing");
535         return d;
536     }
537 
538     foreach (const p; parts.chunks(2)) {
539         const nr = p[0].to!long;
540         bool validUnit;
541         immutable AllUnites = [
542             "msecs", "seconds", "minutes", "hours", "days", "weeks"
543         ];
544         static foreach (Unit; AllUnites) {
545             if (p[1] == Unit) {
546                 d += nr.dur!Unit;
547                 validUnit = true;
548             }
549         }
550         if (!validUnit) {
551             logger.warningf("Invalid unit '%s'. Valid are %-(%s, %).", p[1], AllUnites);
552             return d;
553         }
554     }
555 
556     return d;
557 }