1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dsnapshot.layout_utils;
7
8 import logger = std.experimental.logger;
9 import std.array : empty;
10 import std.exception : collectException;
11
12 import sumtype;
13
14 import dsnapshot.exception;
15 import dsnapshot.layout : Name, Layout;
16 import dsnapshot.types;
17
18 version (unittest) {
19 import unit_threaded.assertions;
20 }
21
22 auto fillLayout(Layout layout_, Flow flow, const RemoteCmd cmd) {
23 import std.algorithm : filter, map, sort;
24 import std.array : array;
25 import std.conv : to;
26 import std.datetime : UTC, DateTimeException, SysTime;
27 import std.file : dirEntries, SpanMode, exists, isDir;
28 import std.path : baseName;
29 import dsnapshot.layout : LSnapshot = Snapshot;
30
31 auto rval = layout_;
32 scope (exit)
33 rval.finalize;
34
35 const names = flow.match!((None a) => null,
36 (FlowRsyncToLocal a) => snapshotNamesFromDir(a.dst.value.Path),
37 (FlowLocal a) => snapshotNamesFromDir(a.dst.value.Path),
38 (FlowLocalToRsync a) => snapshotNamesFromSsh(cmd, a.dst.addr, a.dst.path));
39
40 foreach (const n; names) {
41 try {
42 const t = SysTime.fromISOExtString(n.value, UTC());
43 rval.put(LSnapshot(t, n));
44 } catch (DateTimeException e) {
45 logger.warning("Unable to extract the time from the snapshot name");
46 logger.info(e.msg);
47 logger.info("It is added as a snapshot taken at the time ", SysTime.min);
48 rval.put(LSnapshot(SysTime.min, n));
49 }
50 }
51
52 return rval;
53 }
54
55 Name[] snapshotNamesFromDir(Path dir) {
56 import std.algorithm : filter, map, copy;
57 import std.array : appender;
58 import std.file : dirEntries, SpanMode, exists, isDir;
59 import std.path : baseName;
60
61 if (!dir.toString.exists)
62 return null;
63 if (!dir.toString.isDir)
64 throw new SnapshotException(SnapshotException.DstIsNotADir.init.SnapshotError);
65
66 auto app = appender!(Name[])();
67 dirEntries(dir.toString, SpanMode.shallow).map!(a => a.name)
68 .filter!(a => a.isDir)
69 .map!(a => Name(a.baseName))
70 .copy(app);
71 return app.data;
72 }
73
74 Name[] snapshotNamesFromSsh(const RemoteCmd cmd_, string addr, string path) {
75 import std.algorithm : map, copy;
76 import std.array : appender;
77 import std.process : execute;
78 import std..string : splitLines;
79
80 auto cmd = cmd_.match!((const SshRemoteCmd a) {
81 return a.toCmd(RemoteSubCmd.lsDirs, addr, path);
82 });
83 if (cmd.empty)
84 return null;
85
86 auto res = execute(cmd);
87 if (res.status != 0) {
88 logger.warning(res.output);
89 return null;
90 }
91
92 auto app = appender!(Name[])();
93 res.output.splitLines.map!(a => Name(a)).copy(app);
94
95 return app.data;
96 }
97
98 @("shall scan the directory for all snapshots")
99 unittest {
100 import std.conv : to;
101 import std.datetime : SysTime, UTC, Duration, Clock, dur;
102 import std.file;
103 import std.path;
104 import std.range : enumerate;
105 import sumtype;
106 import dsnapshot.layout : Layout, LayoutConfig, Span;
107
108 immutable tmpDir = "test_snapshot_scan";
109 scope (exit)
110 rmdirRecurse(tmpDir);
111 mkdir(tmpDir);
112
113 auto curr = Clock.currTime;
114 curr.timezone = UTC();
115
116 foreach (const i; 0 .. 15) {
117 mkdir(buildPath(tmpDir, curr.toISOExtString));
118 curr -= 1.dur!"hours";
119 }
120 foreach (const i; 0 .. 15) {
121 curr -= 5.dur!"hours";
122 mkdir(buildPath(tmpDir, curr.toISOExtString));
123 }
124
125 auto conf = LayoutConfig([Span(5, 4.dur!"hours"), Span(5, 1.dur!"days")]);
126 const base = Clock.currTime;
127 auto layout = Layout(base, conf);
128 layout = fillLayout(layout, FlowLocal(LocalAddr(tmpDir), LocalAddr(tmpDir))
129 .Flow, RemoteCmd(SshRemoteCmd.init));
130
131 layout.waiting.length.shouldEqual(0);
132 layout.discarded.length.shouldEqual(20);
133
134 (base - layout.snapshotTimeInBucket(0).get).total!"hours".shouldEqual(4);
135 (base - layout.snapshotTimeInBucket(4).get).total!"hours".shouldEqual(4 * 5);
136 (base - layout.snapshotTimeInBucket(5).get).total!"hours".shouldEqual(4 * 5 + 24 + 1);
137 (base - layout.snapshotTimeInBucket(6).get).total!"hours".shouldEqual(4 * 5 + 24 * 2 + 2);
138 (base - layout.snapshotTimeInBucket(7).get).total!"hours".shouldEqual(4 * 5 + 24 * 3 - 2);
139
140 /// these buckets are filled by the second pass
141 (base - layout.snapshotTimeInBucket(8).get).total!"hours".shouldEqual(4 * 5 + 24 * 4 - 31);
142 (base - layout.snapshotTimeInBucket(9).get).total!"hours".shouldEqual(4 * 5 + 24 * 5 - 60);
143 }