1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module app; 7 8 import logger = std.experimental.logger; 9 import std.algorithm : remove, map, filter; 10 import std.array : array, empty; 11 import std.exception : collectException; 12 import std.path : baseName, expandTilde; 13 import std.stdio : writeln; 14 15 import colorlog; 16 17 import dsnapshot.config; 18 19 int main(string[] args) { 20 import dsnapshot.cmdgroup.admin; 21 import dsnapshot.cmdgroup.backup; 22 import dsnapshot.cmdgroup.remote; 23 import dsnapshot.cmdgroup.restore; 24 import dsnapshot.cmdgroup.watch; 25 26 confLogger(VerboseMode.info); 27 28 auto conf = parseUserArgs(args); 29 30 confLogger(conf.global.verbosity); 31 32 logger.trace(conf.global); 33 static foreach (T; Config.Type.AllowedTypes) { 34 if (auto a = conf.data.peek!T) 35 logger.tracef("%s", *a); 36 } 37 38 if (conf.global.help) { 39 return cmdHelp(conf); 40 } 41 42 import std.variant : visit; 43 44 // dfmt off 45 return conf.data.visit!( 46 (Config.Help a) => cmdHelp(conf), 47 (Config.Backup a) { 48 loadConfig(conf); 49 return cmdBackup(conf.global, a, conf.snapshots); 50 }, 51 (Config.Remotecmd a) => cmdRemote(a), 52 (Config.Restore a) { 53 loadConfig(conf); 54 return cmdRestore(conf.snapshots, a); 55 }, 56 (Config.Verifyconfig a) { 57 loadConfig(conf); 58 logger.info("Done"); 59 return 0; 60 }, 61 (Config.Admin a) { 62 loadConfig(conf); 63 return cmdAdmin(conf.snapshots, a); 64 }, 65 (Config.Watch a) { 66 loadConfig(conf); 67 return cli(conf.global, a, conf.snapshots); 68 } 69 ); 70 // dfmt on 71 } 72 73 @safe: 74 private: 75 76 int cmdHelp(Config conf) @trusted { 77 conf.printHelp; 78 return 0; 79 } 80 81 Config parseUserArgs(string[] args) @trusted { 82 import std.format : format; 83 import std.string : toLower; 84 import std.traits : EnumMembers; 85 static import std.getopt; 86 87 Config conf; 88 conf.data = Config.Help.init; 89 conf.global.progName = args[0].baseName; 90 conf.global.confFile = ".dsnapshot.toml"; 91 92 string group; 93 if (args.length > 1) { 94 group = args[1]; 95 args = args.remove(1); 96 } 97 98 try { 99 void globalParse() { 100 Config.Help data; 101 scope (success) 102 conf.data = data; 103 104 string confFile; 105 // dfmt off 106 data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, 107 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity, 108 "c|config", "Config file to read", &confFile, 109 ); 110 // dfmt on 111 conf.global.help = data.helpInfo.helpWanted; 112 args ~= (conf.global.help ? "-h" : null); 113 114 if (!confFile.empty) 115 conf.global.confFile = confFile.Path; 116 } 117 118 void backupParse() { 119 Config.Backup data; 120 scope (success) 121 conf.data = data; 122 123 string margin; 124 // dfmt off 125 data.helpInfo = std.getopt.getopt(args, 126 "force", "Force a backup to be taken even though it isn't time for it", &data.forceBackup, 127 "ignore-rsync-error-code", "Ignore rsync error code", &data.ignoreRsyncErrorCodes, 128 "margin", "Add a margin when checking if a new snapshot should be taken", &margin, 129 "resume", "If an interrupted backup should be resumed", &data.resume, 130 "s|snapshot", "The name of the snapshot to backup (default: all)", &data.name.value, 131 ); 132 // dfmt on 133 data.newSnapshotMargin = parseDuration(margin); 134 } 135 136 void remotecmdParse() { 137 Config.Remotecmd data; 138 scope (success) 139 conf.data = data; 140 141 // dfmt off 142 data.helpInfo = std.getopt.getopt(args, 143 "cmd", format("Command to execute (%-(%s, %))", [EnumMembers!(RemoteSubCmd)]), &data.cmd, 144 "path", "Path argument for the command", &data.path, 145 ); 146 // dfmt on 147 } 148 149 void restoreParse() { 150 import std.datetime : SysTime, UTC, Clock; 151 152 Config.Restore data; 153 scope (success) 154 conf.data = data; 155 156 string time; 157 // dfmt off 158 data.helpInfo = std.getopt.getopt(args, 159 "delete", "Delete files from dst if they are removed in the snapshot", &data.deleteFromTo, 160 "dst", "Where to restore the snapshot", &data.restoreTo, 161 "s|snapshot", "Name of the snapshot to calculate the disk usage for", &data.name.value, 162 "time", "Pick the snapshot that is closest to this time (default: now). Add a trailing Z if UTC", &time, 163 ); 164 // dfmt on 165 166 try { 167 if (time.length == 0) 168 data.time = Clock.currTime; 169 else 170 data.time = SysTime.fromISOExtString(time); 171 } catch (Exception e) { 172 logger.error(e.msg); 173 throw new Exception("Example of UTC time: 2019-07-18T10:49:29.5765454Z"); 174 } 175 } 176 177 void verifyconfigParse() { 178 conf.data = Config.Verifyconfig.init; 179 } 180 181 void adminParse() { 182 Config.Admin data; 183 scope (success) 184 conf.data = data; 185 186 string[] names; 187 // dfmt off 188 data.helpInfo = std.getopt.getopt(args, 189 "s|snapshot", "Snapshot name (default: all)", &names, 190 "cmd", format!"What to do. Available: %-(%s, %) (default: list)"([EnumMembers!(Config.Admin.Cmd)]), &data.cmd, 191 ); 192 // dfmt on 193 194 data.names = names.map!(a => Name(a)).array; 195 } 196 197 void watchParse() { 198 Config.Watch data; 199 scope (success) 200 conf.data = data; 201 202 // dfmt off 203 data.helpInfo = std.getopt.getopt(args, 204 "max-nr", "Exit after this number of snapshots has been created", &data.maxSnapshots, 205 std.getopt.config.required, "s|snapshot", "The name of the snapshot to backup", &data.name.value, 206 ); 207 // dfmt on 208 } 209 210 alias ParseFn = void delegate(); 211 ParseFn[string] parsers; 212 213 static foreach (T; Config.Type.AllowedTypes) { 214 static if (!is(T == Config.Help)) 215 mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower)); 216 } 217 218 globalParse; 219 220 if (auto p = group in parsers) { 221 (*p)(); 222 } 223 } catch (std.getopt.GetOptException e) { 224 // unknown option 225 conf.global.help = true; 226 logger.error(e.msg); 227 } catch (Exception e) { 228 conf.global.help = true; 229 logger.error(e.msg); 230 } 231 232 return conf; 233 } 234 235 void loadConfig(ref Config conf) @trusted { 236 import std.conv : to; 237 import std.file : exists, readText, isFile; 238 import std.path : dirName, buildPath; 239 import toml; 240 241 scope (exit) 242 logger.trace(conf); 243 244 if (!exists(conf.global.confFile.toString) || !conf.global.confFile.toString.isFile) { 245 logger.errorf("Configuration %s do not exist", conf.global.confFile); 246 return; 247 } 248 249 static auto tryLoading(string configFile) { 250 auto txt = readText(configFile); 251 auto doc = parseTOML(txt); 252 return doc; 253 } 254 255 TOMLDocument doc; 256 try { 257 doc = tryLoading(conf.global.confFile.toString); 258 } catch (Exception e) { 259 logger.warning("Unable to read the configuration from ", conf.global.confFile); 260 logger.warning(e.msg); 261 return; 262 } 263 264 alias Fn = void delegate(ref Config c, ref TOMLValue v); 265 Fn[string] tables; 266 267 tables["snapshot"] = (ref Config c, ref TOMLValue snapshots) { 268 foreach (name, data; snapshots) { 269 SnapshotConfig s; 270 s.name = name; 271 s.layout = makeDefaultLayout; 272 foreach (k, v; data) { 273 try { 274 switch (k) { 275 case "rsync": 276 auto rsync = parseRsync(v, name); 277 s.syncCmd = rsync; 278 break; 279 case "encfs": 280 s.crypt = parseEncfs(v, name); 281 break; 282 case "pre_exec": 283 s.hooks.preExec = v.array.map!(a => a.str).array; 284 break; 285 case "post_exec": 286 s.hooks.postExec = v.array.map!(a => a.str).array; 287 break; 288 case "span": 289 auto layout = parseLayout(v); 290 s.layout = layout; 291 break; 292 case "dsnapshot": 293 s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) { 294 a.dsnapshot = v.str; 295 return a; 296 }); 297 break; 298 case "rsh": 299 s.remoteCmd = s.remoteCmd.match!((SshRemoteCmd a) { 300 a.rsh = v.array.map!(a => a.str).array; 301 return a; 302 }); 303 break; 304 default: 305 logger.infof("Unknown option '%s' in section 'snapshot.%s' in configuration", 306 k, name); 307 } 308 } catch (Exception e) { 309 logger.error(e.msg).collectException; 310 } 311 } 312 c.snapshots ~= s; 313 } 314 }; 315 316 foreach (curr; doc.byKeyValue.filter!(a => a.value.type == TOML_TYPE.TABLE)) { 317 try { 318 if (auto t = curr.key in tables) { 319 (*t)(conf, curr.value); 320 } else { 321 logger.infof("Unknown section '%s' in configuration", curr.key); 322 } 323 } catch (Exception e) { 324 logger.error(e.msg).collectException; 325 } 326 } 327 } 328 329 import toml : TOMLValue; 330 331 auto parseLayout(ref TOMLValue tv) @trusted { 332 import std.algorithm : sort; 333 import std.conv : to; 334 import std.datetime : Clock; 335 import std.range : chunks; 336 import std.string : split; 337 import std.typecons : Nullable; 338 import dsnapshot.layout : Span, LayoutConfig, Layout; 339 340 Span[long] spans; 341 342 static Nullable!Span parseASpan(ref TOMLValue data) { 343 typeof(return) rval; 344 345 immutable intervalKey = "interval"; 346 immutable nrKey = "nr"; 347 if (intervalKey !in data || nrKey !in data) { 348 logger.warning("Missing either 'nr' or 'space' key"); 349 return rval; 350 } 351 352 auto d = parseDuration(data[intervalKey].str); 353 354 const nr = data[nrKey].integer; 355 if (nr <= 0) { 356 logger.warning("nr must be positive"); 357 return rval; 358 } 359 360 rval = Span(cast(uint) nr, d); 361 362 return rval; 363 } 364 365 foreach (key, data; tv) { 366 try { 367 const idx = key.to!long; 368 auto span = parseASpan(data); 369 if (span.isNull) { 370 logger.warning(tv); 371 } else { 372 spans[idx] = span.get; 373 } 374 } catch (Exception e) { 375 logger.warning(e.msg); 376 logger.warning(tv); 377 } 378 } 379 380 return Layout(Clock.currTime, LayoutConfig(spans.byKeyValue 381 .array 382 .sort!((a, b) => a.key < b.key) 383 .map!(a => a.value) 384 .array)); 385 } 386 387 auto parseRsync(ref TOMLValue tv, const string parent) @trusted { 388 import std.format : format; 389 import dsnapshot.types; 390 391 RsyncConfig rval; 392 393 string src, srcAddr, dst, dstAddr; 394 foreach (key, data; tv) { 395 switch (key) { 396 case "src": 397 src = data.str; 398 break; 399 case "src_addr": 400 srcAddr = data.str; 401 break; 402 case "dst": 403 dst = data.str; 404 break; 405 case "dst_addr": 406 dstAddr = data.str; 407 break; 408 case "cmd_rsync": 409 rval.cmdRsync = data.str; 410 break; 411 case "cross_fs": 412 rval.oneFs = data == true; 413 break; 414 case "fakeroot": 415 rval.useFakeRoot = data == true; 416 break; 417 case "rsync_fakeroot_args": 418 rval.rsyncFakerootArgs = data.array.map!(a => a.str).array; 419 break; 420 case "fakeroot_args": 421 rval.fakerootArgs = data.array.map!(a => a.str).array; 422 break; 423 case "link_dest": 424 rval.useLinkDest = data == true; 425 break; 426 case "low_prio": 427 rval.lowPrio = data == true; 428 break; 429 case "exclude": 430 rval.exclude = data.array.map!(a => a.str).array; 431 break; 432 case "rsync_cmd": 433 rval.cmdRsync = data.str; 434 break; 435 case "rsync_rsh": 436 rval.rsh = data.str; 437 break; 438 case "diskusage_cmd": 439 rval.cmdDiskUsage = data.array.map!(a => a.str).array; 440 break; 441 case "rsync_backup_args": 442 rval.backupArgs = data.array.map!(a => a.str).array; 443 break; 444 case "rsync_restore_args": 445 rval.restoreArgs = data.array.map!(a => a.str).array; 446 break; 447 case "progress": 448 rval.progress = data.array.map!(a => a.str).array; 449 break; 450 default: 451 logger.infof("Unknown option '%s' in section 'snapshot.%s.rsync' in configuration", 452 key, parent); 453 } 454 } 455 456 if (srcAddr.empty && dstAddr.empty) { 457 rval.flow = FlowLocal(LocalAddr(src.expandTilde), LocalAddr(dst)); 458 } else if (!srcAddr.empty && dstAddr.empty) { 459 rval.flow = FlowRsyncToLocal(RemoteHost(srcAddr, src), LocalAddr(dst)); 460 } else if (srcAddr.empty && !dstAddr.empty) { 461 rval.flow = FlowLocalToRsync(LocalAddr(src), RemoteHost(dstAddr, dst)); 462 } else { 463 logger.warning("The combination of src, src_addr, dst and dst_addr is not supported. It either has to be local->local, local->remote or remote->local"); 464 } 465 466 return rval; 467 } 468 469 auto parseEncfs(ref TOMLValue tv, const string parent) @trusted { 470 import std.format : format; 471 import dsnapshot.types; 472 473 EncFsConfig rval; 474 475 foreach (key, data; tv) { 476 switch (key) { 477 case "config": 478 rval.configFile = data.str; 479 break; 480 case "encrypted_path": 481 rval.encryptedPath = data.str; 482 break; 483 case "passwd": 484 rval.passwd = data.str; 485 break; 486 case "mount_cmd": 487 rval.mountCmd = data.array.map!(a => a.str).array; 488 break; 489 case "mount_fuse_opts": 490 rval.mountFuseOpts = data.array.map!(a => a.str).array; 491 break; 492 case "unmount_cmd": 493 rval.unmountCmd = data.array.map!(a => a.str).array; 494 break; 495 case "unmount_fuse_opts": 496 rval.unmountFuseOpts = data.array.map!(a => a.str).array; 497 break; 498 default: 499 logger.infof("Unknown option '%s' in section 'snapshot.%s.encfs' in configuration", 500 key, parent); 501 } 502 } 503 504 if (rval.encryptedPath.empty) { 505 logger.error("No 'encrypted_path' specified for ", parent); 506 } 507 508 return rval; 509 } 510 511 /** The default layout to use if none is specified by the user. 512 */ 513 auto makeDefaultLayout() { 514 import std.datetime : Clock, dur; 515 import dsnapshot.layout; 516 517 const base = Clock.currTime; 518 auto conf = LayoutConfig([ 519 Span(6, 4.dur!"hours"), Span(6, 1.dur!"days"), Span(3, 1.dur!"weeks") 520 ]); 521 return Layout(base, conf); 522 } 523 524 auto parseDuration(string timeSpec) { 525 import std.conv : to; 526 import std.string : split; 527 import std.datetime : Duration, dur; 528 import std.range : chunks; 529 530 Duration d; 531 const parts = timeSpec.split; 532 533 if (parts.length % 2 != 0) { 534 logger.warning("Invalid time specification because either the number or unit is missing"); 535 return d; 536 } 537 538 foreach (const p; parts.chunks(2)) { 539 const nr = p[0].to!long; 540 bool validUnit; 541 immutable AllUnites = [ 542 "msecs", "seconds", "minutes", "hours", "days", "weeks" 543 ]; 544 static foreach (Unit; AllUnites) { 545 if (p[1] == Unit) { 546 d += nr.dur!Unit; 547 validUnit = true; 548 } 549 } 550 if (!validUnit) { 551 logger.warningf("Invalid unit '%s'. Valid are %-(%s, %).", p[1], AllUnites); 552 return d; 553 } 554 } 555 556 return d; 557 }