app source code

1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module app;
7 
8 import logger = std.experimental.logger;
9 import std.algorithm : remove;
10 import std.exception : collectException;
11 import std.path;
12 import std.stdio : writeln;
13 import std..string;
14 
15 import colorlog;
16 
17 import dsnapshot.config;
18 
19 int main(string[] args) {
20     import dsnapshot.cmdgroup.backup;
21     import dsnapshot.cmdgroup.diskusage;
22     import dsnapshot.cmdgroup.remote;
23     import dsnapshot.cmdgroup.restore;
24 
25     confLogger(VerboseMode.info);
26 
27     auto conf = parseUserArgs(args);
28 
29     confLogger(conf.global.verbosity);
30 
31     logger.trace(conf.global);
32     static foreach (T; Config.Type.AllowedTypes) {
33         if (auto a = conf.data.peek!T)
34             logger.tracef("%s", *a);
35     }
36 
37     if (conf.global.help) {
38         return cmdHelp(conf);
39     }
40 
41     import std.variant : visit;
42 
43     // dfmt off
44     return conf.data.visit!(
45       (Config.Help a) => cmdHelp(conf),
46       (Config.Backup a) {
47           loadConfig(conf);
48           return cmdBackup(conf.global, a, conf.snapshots);
49       },
50       (Config.Remotecmd a) => cmdRemote(a),
51           (Config.Diskusage a) {
52           loadConfig(conf);
53           return cmdDiskUsage(conf.snapshots, a);
54       },
55       (Config.Restore a) {
56           loadConfig(conf);
57           return cmdRestore(conf.snapshots, a);
58       },
59       (Config.Verifyconfig a) {
60           loadConfig(conf);
61           logger.info("Done");
62           return 0;
63       }
64     );
65     // dfmt on
66 }
67 
68 @safe:
69 private:
70 
71 int cmdHelp(Config conf) @trusted {
72     conf.printHelp;
73     return 0;
74 }
75 
76 Config parseUserArgs(string[] args) @trusted {
77     import std.format : format;
78     import std..string : toLower;
79     import std.traits : EnumMembers;
80     static import std.getopt;
81 
82     Config conf;
83     conf.data = Config.Help.init;
84     conf.global.progName = args[0].baseName;
85     conf.global.confFile = ".dsnapshot.toml";
86 
87     string group;
88     if (args.length > 1) {
89         group = args[1];
90         args = args.remove(1);
91     }
92 
93     try {
94         string confFile;
95 
96         void globalParse() {
97             // dfmt off
98             conf.global.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough,
99                 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity,
100                 "c|config", "Config file to read", &confFile,
101                 );
102             // dfmt on
103         }
104 
105         void backupParse() {
106             Config.Backup data;
107             scope (success)
108                 conf.data = data;
109 
110             // dfmt off
111             data.helpInfo = std.getopt.getopt(args,
112                 "ignore-rsync-error-code", "Ignore rsync error code", &data.ignoreRsyncErrorCodes,
113                 "s|snapshot", "The name of the snapshot to backup (default: all)", &data.name.value,
114                 );
115             // dfmt on
116         }
117 
118         void remotecmdParse() {
119             Config.Remotecmd data;
120             scope (success)
121                 conf.data = data;
122 
123             // dfmt off
124             data.helpInfo = std.getopt.getopt(args,
125                 "cmd", format("Command to execute (%-(%s, %))", [EnumMembers!(RemoteSubCmd)]), &data.cmd,
126                 "path", "Path argument for the command", &data.path,
127                 );
128             // dfmt on
129         }
130 
131         void diskusageParse() {
132             Config.Diskusage data;
133             scope (success)
134                 conf.data = data;
135 
136             // dfmt off
137             data.helpInfo = std.getopt.getopt(args,
138                 "s|snapshot", "Name of the snapshot to calculate the disk usage for", &data.name.value,
139                 );
140             // dfmt on
141         }
142 
143         void restoreParse() {
144             import std.datetime : SysTime, UTC, Clock;
145 
146             Config.Restore data;
147             scope (success)
148                 conf.data = data;
149 
150             string time;
151             // dfmt off
152             data.helpInfo = std.getopt.getopt(args,
153                 "delete", "Delete files from dst if they are removed in the snapshot", &data.deleteFromTo,
154                 "dst", "Where to restore the snapshot", &data.restoreTo,
155                 "s|snapshot", "Name of the snapshot to calculate the disk usage for", &data.name.value,
156                 "time", "Pick the snapshot that is closest to this time (default: now). Add a trailing Z if UTC", &time,
157                 );
158             // dfmt on
159 
160             try {
161                 if (time.length == 0)
162                     data.time = Clock.currTime;
163                 else
164                     data.time = SysTime.fromISOExtString(time);
165             } catch (Exception e) {
166                 logger.error(e.msg);
167                 throw new Exception("Example of UTC time: 2019-07-18T10:49:29.5765454Z");
168             }
169         }
170 
171         void verifyconfigParse() {
172             conf.data = Config.Verifyconfig.init;
173         }
174 
175         alias ParseFn = void delegate();
176         ParseFn[string] parsers;
177 
178         if (confFile.length != 0)
179             conf.global.confFile = confFile.Path;
180 
181         globalParse;
182 
183         static foreach (T; Config.Type.AllowedTypes) {
184             static if (!is(T == Config.Help))
185                 mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower));
186         }
187 
188         conf.global.help = true;
189         if (auto p = group in parsers) {
190             (*p)();
191             conf.global.help = conf.global.helpInfo.helpWanted;
192         }
193     } catch (std.getopt.GetOptException e) {
194         // unknown option
195         conf.global.help = true;
196         logger.error(e.msg);
197     } catch (Exception e) {
198         conf.global.help = true;
199         logger.error(e.msg);
200     }
201 
202     return conf;
203 }
204 
205 void loadConfig(ref Config conf) @trusted {
206     import std.algorithm : filter, map;
207     import std.array : array;
208     import std.conv : to;
209     import std.file : exists, readText, isFile;
210     import std.path : dirName, buildPath;
211     import toml;
212 
213     scope (exit)
214         logger.trace(conf);
215 
216     if (!exists(conf.global.confFile.toString) || !conf.global.confFile.toString.isFile) {
217         logger.errorf("Configuration %s do not exist", conf.global.confFile);
218         return;
219     }
220 
221     static auto tryLoading(string configFile) {
222         auto txt = readText(configFile);
223         auto doc = parseTOML(txt);
224         return doc;
225     }
226 
227     TOMLDocument doc;
228     try {
229         doc = tryLoading(conf.global.confFile.toString);
230     } catch (Exception e) {
231         logger.warning("Unable to read the configuration from ", conf.global.confFile);
232         logger.warning(e.msg);
233         return;
234     }
235 
236     alias Fn = void delegate(ref Config c, ref TOMLValue v);
237     Fn[string] tables;
238 
239     tables["snapshot"] = (ref Config c, ref TOMLValue snapshots) {
240         foreach (name, data; snapshots) {
241             Snapshot s;
242             s.name = name;
243             s.layout = makeDefaultLayout;
244             foreach (k, v; data) {
245                 try {
246                     switch (k) {
247                     case "rsync":
248                         auto rsync = parseRsync(v, name);
249                         s.syncCmd = rsync;
250                         break;
251                     case "pre_exec":
252                         s.hooks.preExec = v.array.map!(a => a.str).array;
253                         break;
254                     case "post_exec":
255                         s.hooks.postExec = v.array.map!(a => a.str).array;
256                         break;
257                     case "span":
258                         auto layout = parseLayout(v);
259                         s.layout = layout;
260                         break;
261                     case "dsnapshot":
262                         s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) {
263                             a.dsnapshot = v.str;
264                             return a;
265                         });
266                         break;
267                     case "rsh":
268                         s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) {
269                             a.rsh = v.array.map!(a => a.str).array;
270                             return a;
271                         });
272                         break;
273                     default:
274                         logger.infof("Unknown option '%s' in section 'snapshot.%s' in configuration",
275                                 k, name);
276                     }
277                 } catch (Exception e) {
278                     logger.error(e.msg).collectException;
279                 }
280             }
281             c.snapshots ~= s;
282         }
283     };
284 
285     foreach (curr; doc.byKeyValue.filter!(a => a.value.type == TOML_TYPE.TABLE)) {
286         try {
287             if (auto t = curr.key in tables) {
288                 (*t)(conf, curr.value);
289             } else {
290                 logger.infof("Unknown section '%s' in configuration", curr.key);
291             }
292         } catch (Exception e) {
293             logger.error(e.msg).collectException;
294         }
295     }
296 }
297 
298 import toml : TOMLValue;
299 
300 auto parseLayout(ref TOMLValue tv) @trusted {
301     import std.algorithm : sort, map;
302     import std.array : array;
303     import std.conv : to;
304     import std.datetime : Duration, dur, Clock;
305     import std.range : chunks;
306     import std..string : split;
307     import std.typecons : Nullable;
308     import dsnapshot.layout : Span, LayoutConfig, Layout;
309 
310     Span[long] spans;
311 
312     static Nullable!Span parseASpan(ref TOMLValue data) {
313         typeof(return) rval;
314 
315         immutable intervalKey = "interval";
316         immutable nrKey = "nr";
317         if (intervalKey !in data || nrKey !in data) {
318             logger.warning("Missing either 'nr' or 'space' key");
319             return rval;
320         }
321 
322         const parts = data[intervalKey].str.split;
323         if (parts.length % 2 != 0) {
324             logger.warning(
325                     "Invalid space specification because either the number or unit is missing");
326             return rval;
327         }
328 
329         Duration d;
330         foreach (const p; parts.chunks(2)) {
331             const nr = p[0].to!long;
332             bool validUnit;
333             immutable AllUnites = [
334                 "msecs", "seconds", "minutes", "hours", "days", "weeks"
335             ];
336             static foreach (Unit; AllUnites) {
337                 if (p[1] == Unit) {
338                     d += nr.dur!Unit;
339                     validUnit = true;
340                 }
341             }
342             if (!validUnit) {
343                 logger.warningf("Invalid unit '%s'. Valid are %-(%s, %).", p[1], AllUnites);
344                 return rval;
345             }
346         }
347 
348         const nr = data[nrKey].integer;
349         if (nr <= 0) {
350             logger.warning("nr must be positive");
351             return rval;
352         }
353 
354         rval = Span(cast(uint) nr, d);
355 
356         return rval;
357     }
358 
359     foreach (key, data; tv) {
360         try {
361             const idx = key.to!long;
362             auto span = parseASpan(data);
363             if (span.isNull) {
364                 logger.warning(tv);
365             } else {
366                 spans[idx] = span.get;
367             }
368         } catch (Exception e) {
369             logger.warning(e.msg);
370             logger.warning(tv);
371         }
372     }
373 
374     return Layout(Clock.currTime, LayoutConfig(spans.byKeyValue
375             .array
376             .sort!((a, b) => a.key < b.key)
377             .map!(a => a.value)
378             .array));
379 }
380 
381 auto parseRsync(ref TOMLValue tv, const string parent) @trusted {
382     import std.algorithm : map;
383     import std.array : array, empty;
384     import std.format : format;
385     import dsnapshot.types;
386 
387     RsyncConfig rval;
388 
389     string src, srcAddr, dst, dstAddr;
390     foreach (key, data; tv) {
391         switch (key) {
392         case "src":
393             src = data.str;
394             break;
395         case "src_addr":
396             srcAddr = data.str;
397             break;
398         case "dst":
399             dst = data.str;
400             break;
401         case "dst_addr":
402             dstAddr = data.str;
403             break;
404         case "cmd_rsync":
405             rval.cmdRsync = data.str;
406             break;
407         case "cross_fs":
408             rval.oneFs = data == true;
409             break;
410         case "fakeroot":
411             rval.useFakeRoot = data == true;
412             break;
413         case "link_dest":
414             rval.useLinkDest = data == true;
415             break;
416         case "low_prio":
417             rval.lowPrio = data == true;
418             break;
419         case "exclude":
420             rval.exclude = data.array.map!(a => a.str).array;
421             break;
422         case "rsync_cmd":
423             rval.cmdRsync = data.str;
424             break;
425         case "diskusage_cmd":
426             rval.cmdDiskUsage = data.array.map!(a => a.str).array;
427             break;
428         case "rsync_args":
429             rval.args = data.array.map!(a => a.str).array;
430             break;
431         case "progress":
432             rval.progress = data.array.map!(a => a.str).array;
433             break;
434         default:
435             logger.infof("Unknown option '%s' in section 'snapshot.%s.rsync' in configuration",
436                     key, parent);
437         }
438     }
439 
440     if (srcAddr.empty && dstAddr.empty) {
441         rval.flow = FlowLocal(LocalAddr(src.expandTilde), LocalAddr(dst));
442     } else if (!srcAddr.empty && dstAddr.empty) {
443         rval.flow = FlowRsyncToLocal(RsyncAddr(srcAddr, src), LocalAddr(dst));
444     } else if (srcAddr.empty && !dstAddr.empty) {
445         rval.flow = FlowLocalToRsync(LocalAddr(src), RsyncAddr(dstAddr, dst));
446     } else {
447         logger.warning("The combination of src, src_addr, dst and dst_addr is not supported. It either has to be local->local, local->remote, remote->local");
448     }
449 
450     return rval;
451 }
452 
453 /** The default layout to use if none is specified by the user.
454  */
455 auto makeDefaultLayout() {
456     import std.datetime : Clock, dur;
457     import dsnapshot.layout;
458 
459     const base = Clock.currTime;
460     auto conf = LayoutConfig([
461             Span(6, 4.dur!"hours"), Span(6, 1.dur!"days"), Span(3, 1.dur!"weeks")
462             ]);
463     return Layout(base, conf);
464 }