1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module dsnapshot.layout_utils; 7 8 import logger = std.experimental.logger; 9 import std.array : empty; 10 import std.exception : collectException; 11 12 import sumtype; 13 14 import dsnapshot.exception; 15 import dsnapshot.layout : Name, Layout; 16 import dsnapshot.types; 17 18 version (unittest) { 19 import unit_threaded.assertions; 20 } 21 22 @safe: 23 24 Layout fillLayout(const Layout layout_, const Flow flow, const RemoteCmd cmd) { 25 import std.algorithm : filter, map, sort; 26 import std.array : array; 27 import std.conv : to; 28 import std.datetime : UTC, DateTimeException, SysTime; 29 import std.file : dirEntries, SpanMode, exists, isDir; 30 import std.path : baseName; 31 import dsnapshot.layout : LSnapshot = Snapshot; 32 33 Layout rval = layout_.dup; 34 35 const names = flow.match!((None a) => null, 36 (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path), 37 (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path), 38 (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path)); 39 40 foreach (const n; names) { 41 try { 42 const t = SysTime.fromISOExtString(n.value, UTC()); 43 rval.put(LSnapshot(t, n)); 44 } catch (DateTimeException e) { 45 logger.warning("Unable to extract the time from the snapshot name"); 46 logger.info(e.msg); 47 logger.info("It is added as a snapshot taken at the time ", SysTime.min); 48 rval.put(LSnapshot(SysTime.min, n)); 49 } 50 } 51 52 rval.finalize; 53 54 return rval; 55 } 56 57 Name[] snapshotNamesFromDir(Path dir) @trusted { 58 import std.algorithm : filter, map, copy; 59 import std.array : appender; 60 import std.file : dirEntries, SpanMode, exists, isDir; 61 import std.path : baseName; 62 63 if (!dir.toString.exists) 64 return null; 65 if (!dir.toString.isDir) 66 throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError); 67 68 auto app = appender!(Name[])(); 69 dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name) 70 .filter!(a => a.isDir) 71 .filter!(a => a.baseName != snapshotLatest) 72 .map!(a => Name(a.baseName)) 73 .copy(app); 74 return app.data; 75 } 76 77 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) { 78 import std.algorithm : map, copy, filter; 79 import std.array : appender; 80 import std.path : baseName; 81 import std.process : execute; 82 import std.string : splitLines; 83 84 auto cmd = cmd_.match!((const SshRemoteCmd a) { 85 return a.toCmd(RemoteSubCmd.lsDirs, addr, path); 86 }); 87 if (cmd.empty) 88 return null; 89 90 auto res = execute(cmd); 91 if (res.status != 0) { 92 logger.warning(res.output); 93 return null; 94 } 95 96 auto app = appender!(Name[])(); 97 res.output 98 .splitLines 99 .filter!(a => a.baseName != snapshotLatest) 100 .map!(a => Name(a)) 101 .copy(app); 102 103 return app.data; 104 } 105 106 @("shall scan the directory for all snapshots") 107 unittest { 108 import std.conv : to; 109 import std.datetime : SysTime, UTC, Duration, Clock, dur, Interval; 110 import std.file; 111 import std.path; 112 import std.range : enumerate; 113 import sumtype; 114 import dsnapshot.layout : Layout, LayoutConfig, Span; 115 116 immutable tmpDir = "test_snapshot_scan"; 117 scope (exit) 118 () @trusted { rmdirRecurse(tmpDir); }(); 119 () @trusted { rmdirRecurse(tmpDir).collectException; }(); 120 mkdir(tmpDir).collectException; 121 122 const offset = 5.dur!"minutes"; 123 const base = Clock.currTime.toUTC; 124 auto curr = cast() base; 125 126 foreach (const i; 0 .. 15) { 127 mkdir(buildPath(tmpDir, curr.toISOExtString)); 128 curr -= 4.dur!"hours"; 129 } 130 foreach (const i; 0 .. 4) { 131 curr -= 8.dur!"hours"; 132 mkdir(buildPath(tmpDir, curr.toISOExtString)); 133 } 134 135 auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]); 136 auto layout = Layout(base, conf); 137 layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir)) 138 .Flow, RemoteCmd(SshRemoteCmd.init)); 139 140 layout.discarded.length.shouldEqual(10); 141 142 (base - layout.snapshotTimeInBucket(0).get).total!"minutes".shouldEqual(0); 143 (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(3 * 5 + 1); 144 (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5); 145 (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 1); 146 (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 2); 147 /// these buckets are filled by the second pass 148 (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 3); 149 layout.snapshotTimeInBucket(9).isNull.shouldBeTrue; 150 }